Work in progress on integrating build server

.gitignore

@@ -6,3 +6,4 @@ tmp/
 build_*/
 *~
 *.pyc
+buildserver.box

README.buildserver

@@ -0,0 +1,17 @@
+
+Setting up a build server:
+
+1. Install VirtualBox, vagrant and vagrant-snap
+2. In the buildserver directory, run 'vagrant up'. Will take a long time.
+3. Log in with 'vagrant ssh'
+4. Check it all looks ok, then 'sudo shutdown -h now'
+5. Back in the main directory, run 'VBoxManage listvms' look for
+   buildserver_xxxx
+6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'.
+   Will take a while.
+7. You should now have a new 'buildserver.box'
+
+You should now be able to use the --server option on build.py and builds will
+take place in the clean, secure, isolated environment of a fresh virtual
+machine for each app built.
+

build.py

@@ -24,6 +24,7 @@ import subprocess
 import re
 import zipfile
 import tarfile
+import traceback
 from xml.dom.minidom import Document
 from optparse import OptionParser
 
@@ -44,8 +45,18 @@ parser.add_option("-s", "--stop", action="store_true", default=False,
                   help="Make the build stop on exceptions")
 parser.add_option("-t", "--test", action="store_true", default=False,
                   help="Test mode - put output in the tmp directory only.")
+parser.add_option("--server", action="store_true", default=False,
+                  help="Use build server")
+parser.add_option("--on-server", action="store_true", default=False,
+                  help="Specify that we're running on the build server")
+parser.add_option("-f", "--force", action="store_true", default=False,
+                  help="Force build of disabled app. Only allowed in test mode.")
 (options, args) = parser.parse_args()
 
+if options.force and not options.test:
+    print "Force is only allowed in test mode"
+    sys.exit(1)
+
 # Get all apps...
 apps = common.read_metadata(options.verbose)
 
@@ -84,7 +95,7 @@ for app in apps:
     if options.package and options.package != app['id']:
         # Silent skip...
         pass
-    elif app['Disabled']:
+    elif app['Disabled'] and not options.force:
         if options.verbose:
             print "Skipping %s: disabled" % app['id']
     elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
@@ -107,7 +118,7 @@ for app in apps:
                 dest_repo = os.path.join(repo_dir, app['id'] + '_' +
                         thisbuild['vercode'] + '.apk')
 
-                if os.path.exists(dest) or os.path.exists(dest_repo):
+                if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
                     if options.verbose:
                         print "..version " + thisbuild['version'] + " already exists"
                 elif thisbuild['commit'].startswith('!'):
@@ -121,6 +132,74 @@ for app in apps:
                         mstart = 'Building version '
                     print mstart + thisbuild['version'] + ' of ' + app['id']
 
+                    if options.server:
+
+                        import paramiko
+
+                        # Start up the virtual maachine...
+                        if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
+                            # Not a very helpful message yet!
+                            raise BuildException("Failed to set up build server")
+                        # Get SSH configuration settings for us to connect...
+                        subprocess.call('vagrant ssh-config >sshconfig',
+                                cwd='builder', shell=True)
+                        vagranthost = 'default' # Host in ssh config file
+
+                        # Load and parse the SSH config...
+                        sshconfig = paramiko.SSHConfig()
+                        sshconfig.parse('builder/sshconfig')
+                        sshconfig = sshconfig.lookup(vagranthost)
+
+                        # Open SSH connection...
+                        ssh = paramiko.SSHClient()
+                        ssh.set_missing_host_key_policy(paramiko.AcceptPolicy)
+                        ssh.connect(sshconfig['HostName'], username=sshconfig['Username'],
+                            port=sshconfig['Port'], timeout=10, look_for_keys=False,
+                            key_filename=sshconfig['IdentityFile'])
+
+                        # Get an SFTP connection...
+                        ftp = ssh.open_sftp()
+                        ftp.get_channel().settimeout(15)
+
+                        # Put all the necessary files in place...
+                        ftp.chdir('/home/vagrant')
+                        ftp.put('build.py', 'build.py')
+                        ftp.put('common.py', 'common.py')
+                        ftp.put('config.buildserver.py', 'config.py')
+                        ftp.mkdir('build')
+                        ftp.chdir('build')
+                        ftp.mkdir('extlib')
+                        def send_dir(path):
+                            lastdir = path
+                            for r, d, f in os.walk(base):
+                                while lastdir != os.path.commonprefix([lastdir, root]):
+                                    ftp.chdir('..')
+                                    lastdir = os.path.split(lastdir)[0]
+                                lastdir = r
+                                for ff in f:
+                                    ftp.put(os.path.join(r, ff), ff)
+                        ftp.send_dir(app['id'])
+                        # TODO: send relevant extlib directories too
+                        ftp.chdir('/home/vagrant')
+
+                        # Execute the build script...
+                        ssh.exec_command('python build.py --on-server -p ' +
+                                app['id'])
+
+                        # Retrieve the built files...
+                        apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
+                        tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
+                        ftp.chdir('unsigned')
+                        ftp.get(apkfile, os.path.join(output_dir, apkfile))
+                        ftp.get(tarball, os.path.join(output_dir, tarball))
+
+                        # Get rid of the virtual machine...
+                        if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
+                            # Not a very helpful message yet!
+                            raise BuildException("Failed to destroy")
+
+                    else:
+
                         # Prepare the source code...
                         root_dir = common.prepare_source(vcs, app, thisbuild,
                                 build_dir, extlib_dir, sdk_path, ndk_path,
@@ -128,7 +207,7 @@ for app in apps:
                         refreshed_source = True
 
                         # Scan before building...
-                    buildprobs = common.scan_source(build_dir)
+                        buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
                         if len(buildprobs) > 0:
                             print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
                             for problem in buildprobs:
@@ -264,7 +343,7 @@ for app in apps:
             except Exception as e:
                 if options.stop:
                     raise
-                print "Could not build app %s due to unknown error: %s" % (app['id'], e)
+                print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
                 failed_apps[app['id']] = e
 
 for app in build_succeeded:

builder/.gitignore

@@ -0,0 +1 @@
+sshconfig

builder/Vagrantfile

@@ -0,0 +1,8 @@
+Vagrant::Config.run do |config|
+
+  config.vm.box = "buildserver"
+  config.vm.box_url = "../buildserver.box"
+
+  config.vm.customize ["modifyvm", :id, "--memory", "768"]
+
+end

buildserver/.gitignore

@@ -0,0 +1 @@
+.vagrant

buildserver/Vagrantfile

@@ -0,0 +1,26 @@
+Vagrant::Config.run do |config|
+
+  config.vm.box = "debian6-32"
+  config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box"
+
+  config.vm.customize ["modifyvm", :id, "--memory", "1024"]
+
+  config.vm.provision :shell, :path => "fixpaths.sh"
+  # Set apt proxy - remove, or adjust this, accordingly!
+  config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://thurlow:3142\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
+
+  config.vm.provision :chef_solo do |chef|
+    chef.cookbooks_path = "cookbooks"
+    chef.log_level = :debug 
+    chef.json = {
+      :settings => {
+        :sdk_loc => "/home/vagrant/android-sdk",
+        :ndk_loc => "/home/vagrant/android-ndk",
+        :user => "vagrant"
+      }
+    }
+    chef.add_recipe "android-sdk"
+    chef.add_recipe "android-ndk"
+    chef.add_recipe "fdroidbuild-general"
+  end
+end

buildserver/cookbooks/android-ndk/recipes/default.rb

@@ -0,0 +1,17 @@
+
+ndk_loc = node[:settings][:ndk_loc]
+
+script "setup-android-ndk" do
+  interpreter "bash"
+  user node[:settings][:user]
+  cwd "/tmp"
+  code "
+    wget http://dl.google.com/android/ndk/android-ndk-r7-linux-x86.tar.bz2
+    tar jxvf android-ndk-r7-linux-x86.tar.bz2
+    mv android-ndk-r7 #{ndk_loc}
+  "
+  not_if do
+    File.exists?("#{ndk_loc}")
+  end
+end
+

buildserver/cookbooks/android-sdk/recipes/default.rb

@@ -0,0 +1,34 @@
+%w{openjdk-6-jdk}.each do |pkg|
+  package pkg do
+    action :install
+  end
+end
+
+sdk_loc = node[:settings][:sdk_loc]
+user = node[:settings][:user]
+
+script "setup-android-sdk" do
+  interpreter "bash"
+  user user
+  cwd "/tmp"
+  code "
+    wget http://dl.google.com/android/android-sdk_r16-linux.tgz
+    tar zxvf android-sdk_r16-linux.tgz
+    mv android-sdk-linux #{sdk_loc}
+    rm android-sdk_r16-linux.tgz
+    #{sdk_loc}/tools/android update sdk --no-ui -t platform-tool
+    #{sdk_loc}/tools/android update sdk --no-ui -t platform
+    #{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool
+  "
+  not_if do
+    File.exists?("#{sdk_loc}")
+  end
+end
+
+execute "add-android-sdk-path" do
+  user user
+  path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools"
+  command "echo \"export PATH=\\$PATH:#{path}\" >> /home/#{user}/.bashrc"
+  not_if "grep #{sdk_loc} /home/#{user}/.bashrc"
+end
+

buildserver/cookbooks/fdroidbuild-general/recipes/default.rb

@@ -0,0 +1,7 @@
+
+%w{ant ant-contrib maven2 javacc python}.each do |pkg|
+  package pkg do
+    action :install
+  end
+end
+

buildserver/fixpaths.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+fixit()
+{
+  #Fix sudoers so the PATH gets passed through, otherwise chef
+  #provisioning doesn't work.
+  if [ -z "$1" ]; then
+    export EDITOR=$0 && sudo -E visudo
+  else
+    echo "Fix sudoers"
+    echo "Defaults exempt_group=admin" >> $1
+  fi
+  #Stick the gems bin onto root's path as well.
+  sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc
+  # Restart sudo so it gets the changes straight away
+  sudo /etc/init.d/sudo restart
+}
+
+sudo grep "exempt_group" /etc/sudoers -q
+if [ "$?" -eq "1" ]; then
+  fixit
+fi
+

common.py

@@ -892,7 +892,7 @@ def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, j
 
 # Scan the source code in the given directory (and all subdirectories)
 # and return a list of potential problems.
-def scan_source(source_dir):
+def scan_source(build_dir, root_dir, thisbuild):
 
     problems = []
 

config.buildserver.py

@@ -0,0 +1,4 @@
+aapt_path = "/home/vagrant/android-sdk/platform-tools/aapt"
+sdk_path = "/home/vagrant/android-sdk"
+ndk_path = "/home/vagrant/android-ndk"
+javacc_path = "/usr/share/java"

scanner.py

@@ -93,7 +93,7 @@ for app in apps:
                     refreshed_source = True
 
                     # Do the scan...
-                    buildprobs = common.scan_source(build_dir)
+                    buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
                     for problem in buildprobs:
                         problems.append(problem + 
                             ' in ' + app['id'] + ' ' + thisbuild['version'])