Latest buildserver wip

README.buildserver

@@ -1,15 +1,14 @@
 
+Integrating the build server setup into the main scripts is a work in progress. Some things may
+not work properly yet. Talk to CiaranG if you're trying to use this and have problems.
+
 Setting up a build server:
 
 1. Install VirtualBox, vagrant and vagrant-snap
-2. In the buildserver directory, run 'vagrant up'. Will take a long time.
+2. Create (or get - ask CiaranG, or wait until I replace this with a download link!) a standard
-3. Log in with 'vagrant ssh'
+   Debian Squeeze vagrant-compatible base box called 'debian6-32'
-4. Check it all looks ok, then 'sudo shutdown -h now'
+3. Run makebuildserver.sh. This will take a long time. The end result is a new base box called
-5. Back in the main directory, run 'VBoxManage listvms' look for
+   'buildserver'.
-   buildserver_xxxx
-6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'.
-   Will take a while.
-7. You should now have a new 'buildserver.box'
 
 You should now be able to use the --server option on build.py and builds will
 take place in the clean, secure, isolated environment of a fresh virtual

build.py

@@ -32,110 +32,18 @@ import common
 from common import BuildException
 from common import VCSException
 
-#Read configuration...
-execfile('config.py')
-
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
-                  help="Spew out even more information than normal")
-parser.add_option("-p", "--package", default=None,
-                  help="Build only the specified package")
-parser.add_option("-s", "--stop", action="store_true", default=False,
-                  help="Make the build stop on exceptions")
-parser.add_option("-t", "--test", action="store_true", default=False,
-                  help="Test mode - put output in the tmp directory only.")
-parser.add_option("--server", action="store_true", default=False,
-                  help="Use build server")
-parser.add_option("--on-server", action="store_true", default=False,
-                  help="Specify that we're running on the build server")
-parser.add_option("-f", "--force", action="store_true", default=False,
-                  help="Force build of disabled app. Only allowed in test mode.")
-(options, args) = parser.parse_args()
-
-if options.force and not options.test:
-    print "Force is only allowed in test mode"
-    sys.exit(1)
-
-# Get all apps...
-apps = common.read_metadata(options.verbose)
-
-failed_apps = {}
-build_succeeded = []
-
-log_dir = 'logs'
-if not os.path.isdir(log_dir):
-    print "Creating log directory"
-    os.makedirs(log_dir)
-
-tmp_dir = 'tmp'
-if not os.path.isdir(tmp_dir):
-    print "Creating temporary directory"
-    os.makedirs(tmp_dir)
-
-if options.test:
-    output_dir = tmp_dir
-else:
-    output_dir = 'unsigned'
-    if not os.path.isdir(output_dir):
-        print "Creating output directory"
-        os.makedirs(output_dir)
-
-repo_dir = 'repo'
-
-build_dir = 'build'
-if not os.path.isdir(build_dir):
-    print "Creating build directory"
-    os.makedirs(build_dir)
-extlib_dir = os.path.join(build_dir, 'extlib')
-
-# Build applications...
-for app in apps:
-
-    if options.package and options.package != app['id']:
-        # Silent skip...
-        pass
-    elif app['Disabled'] and not options.force:
-        if options.verbose:
-            print "Skipping %s: disabled" % app['id']
-    elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
-        if options.verbose:
-            print "Skipping %s: no builds specified" % app['id']
-    else:
-
-        build_dir = 'build/' + app['id']
-
-        # Set up vcs interface and make sure we have the latest code...
-        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
-
-        refreshed_source = False
-
-
-        for thisbuild in app['builds']:
-            try:
-                dest = os.path.join(output_dir, app['id'] + '_' +
-                        thisbuild['vercode'] + '.apk')
-                dest_repo = os.path.join(repo_dir, app['id'] + '_' +
-                        thisbuild['vercode'] + '.apk')
-
-                if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
-                    if options.verbose:
-                        print "..version " + thisbuild['version'] + " already exists"
-                elif thisbuild['commit'].startswith('!'):
-                    if options.verbose:
-                        print ("..skipping version " + thisbuild['version'] + " - " +
-                            thisbuild['commit'][1:])
-                else:
-                    if options.verbose:
-                        mstart = '.. building version '
-                    else:
-                        mstart = 'Building version '
-                    print mstart + thisbuild['version'] + ' of ' + app['id']
-
-                    if options.server:
 
+# Do a build on the build server.
+def build_server(app, thisbuild, build_dir, output_dir):
     import paramiko
 
+    # Destroy the builder vm if it already exists...
+    # TODO: need to integrate the snapshot stuff so it doesn't have to
+    # keep wasting time doing this unnecessarily.
+    if os.path.exists(os.path.join('builder', '.vagrant')):
+        if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
+            raise BuildException("Failed to destroy build server")
+
     # Start up the virtual maachine...
     if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
         # Not a very helpful message yet!
@@ -169,30 +77,37 @@ for app in apps:
     ftp.put('build.py', 'build.py')
     ftp.put('common.py', 'common.py')
     ftp.put('config.buildserver.py', 'config.py')
+    ftp.mkdir('metadata')
+    ftp.chdir('metadata')
+    ftp.put(os.path.join('metadata', app['id'] + '.txt'),
+            app['id'] + '.txt')
+    ftp.chdir('..')
     ftp.mkdir('build')
     ftp.chdir('build')
     ftp.mkdir('extlib')
+    ftp.mkdir(app['id'])
+    ftp.chdir('..')
     def send_dir(path):
         lastdir = path
-                            for r, d, f in os.walk(base):
+        for r, d, f in os.walk(path):
-                                while lastdir != os.path.commonprefix([lastdir, root]):
+            ftp.chdir(r)
-                                    ftp.chdir('..')
+            for dd in d:
-                                    lastdir = os.path.split(lastdir)[0]
+                ftp.mkdir(dd)
-                                lastdir = r
             for ff in f:
                 ftp.put(os.path.join(r, ff), ff)
-                        send_dir(app['id'])
+            for i in range(len(r.split('/'))):
-                        # TODO: send relevant extlib directories too
+                ftp.chdir('..')
-                        ftp.chdir('/home/vagrant')
+    send_dir(build_dir)
+    # TODO: send relevant extlib and srclib directories too
 
     # Execute the build script...
     ssh.exec_command('python build.py --on-server -p ' +
-                                app['id'])
+            app['id'] + ' --vercode ' + thisbuild['vercode'])
 
     # Retrieve the built files...
     apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
     tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
-                        ftp.chdir('unsigned')
+    ftp.chdir('/home/vagrant/unsigned')
     ftp.get(apkfile, os.path.join(output_dir, apkfile))
     ftp.get(tarball, os.path.join(output_dir, tarball))
 
@@ -201,13 +116,14 @@ for app in apps:
         # Not a very helpful message yet!
         raise BuildException("Failed to destroy")
 
-                    else:
+
+# Do a build locally.
+def build_local(app, thisbuild, build_dir, output_dir):
 
     # Prepare the source code...
     root_dir = common.prepare_source(vcs, app, thisbuild,
             build_dir, extlib_dir, sdk_path, ndk_path,
-                                javacc_path, not refreshed_source)
+            javacc_path)
-                        refreshed_source = True
 
     # Scan before building...
     buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
@@ -329,6 +245,116 @@ for app in apps:
         shutil.move(os.path.join(tmp_dir, tarfilename),
             os.path.join(output_dir, tarfilename))
 
+
+#Read configuration...
+execfile('config.py')
+
+# Parse command line...
+parser = OptionParser()
+parser.add_option("-v", "--verbose", action="store_true", default=False,
+                  help="Spew out even more information than normal")
+parser.add_option("-p", "--package", default=None,
+                  help="Build only the specified package")
+parser.add_option("-c", "--vercode", default=None,
+                  help="Build only the specified version code")
+parser.add_option("-s", "--stop", action="store_true", default=False,
+                  help="Make the build stop on exceptions")
+parser.add_option("-t", "--test", action="store_true", default=False,
+                  help="Test mode - put output in the tmp directory only.")
+parser.add_option("--server", action="store_true", default=False,
+                  help="Use build server")
+parser.add_option("--on-server", action="store_true", default=False,
+                  help="Specify that we're running on the build server")
+parser.add_option("-f", "--force", action="store_true", default=False,
+                  help="Force build of disabled app. Only allowed in test mode.")
+(options, args) = parser.parse_args()
+
+if options.force and not options.test:
+    print "Force is only allowed in test mode"
+    sys.exit(1)
+
+# Get all apps...
+apps = common.read_metadata(options.verbose)
+
+failed_apps = {}
+build_succeeded = []
+
+log_dir = 'logs'
+if not os.path.isdir(log_dir):
+    print "Creating log directory"
+    os.makedirs(log_dir)
+
+tmp_dir = 'tmp'
+if not os.path.isdir(tmp_dir):
+    print "Creating temporary directory"
+    os.makedirs(tmp_dir)
+
+if options.test:
+    output_dir = tmp_dir
+else:
+    output_dir = 'unsigned'
+    if not os.path.isdir(output_dir):
+        print "Creating output directory"
+        os.makedirs(output_dir)
+
+repo_dir = 'repo'
+
+build_dir = 'build'
+if not os.path.isdir(build_dir):
+    print "Creating build directory"
+    os.makedirs(build_dir)
+extlib_dir = os.path.join(build_dir, 'extlib')
+
+# Filter apps and build versions according to command-line options...
+if options.package:
+    apps = [app for app in apps if app['id'] == options.package]
+if options.vercode:
+    for app in apps:
+        app['builds'] = [b for b in app['builds']
+                if str(b['vercode']) == options.vercode]
+
+# Build applications...
+for app in apps:
+
+    if app['Disabled'] and not options.force:
+        if options.verbose:
+            print "Skipping %s: disabled" % app['id']
+    elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
+        if options.verbose:
+            print "Skipping %s: no builds specified" % app['id']
+    else:
+
+        build_dir = 'build/' + app['id']
+
+        # Set up vcs interface and make sure we have the latest code...
+        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+
+        for thisbuild in app['builds']:
+            try:
+                dest = os.path.join(output_dir, app['id'] + '_' +
+                        thisbuild['vercode'] + '.apk')
+                dest_repo = os.path.join(repo_dir, app['id'] + '_' +
+                        thisbuild['vercode'] + '.apk')
+
+                if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
+                    if options.verbose:
+                        print "..version " + thisbuild['version'] + " already exists"
+                elif thisbuild['commit'].startswith('!'):
+                    if options.verbose:
+                        print ("..skipping version " + thisbuild['version'] + " - " +
+                            thisbuild['commit'][1:])
+                else:
+                    if options.verbose:
+                        mstart = '.. building version '
+                    else:
+                        mstart = 'Building version '
+                    print mstart + thisbuild['version'] + ' of ' + app['id']
+
+                    if options.server:
+                        build_server(app, thisbuild, build_dir, output_dir)
+                    else:
+                        build_local(app, thisbuild, build_dir, output_dir)
+
                     build_succeeded.append(app)
             except BuildException as be:
                 if options.stop:

builder/.gitignore

@@ -1,2 +1,3 @@
 sshconfig
 .vagrant
+*.log

builder/Vagrantfile

@@ -1,7 +1,6 @@
 Vagrant::Config.run do |config|
 
   config.vm.box = "buildserver"
-  config.vm.box_url = "../buildserver.box"
 
   config.vm.customize ["modifyvm", :id, "--memory", "768"]
 

buildserver/cookbooks/fdroidbuild-general/recipes/default.rb

@@ -1,5 +1,5 @@
 
-%w{ant ant-contrib maven2 javacc python}.each do |pkg|
+%w{ant ant-contrib maven2 javacc python git-core mercurial subversion bzr}.each do |pkg|
   package pkg do
     action :install
   end

common.py

@@ -657,10 +657,9 @@ def getsrclib(spec, extlib_dir):
 #  'sdk_path'    - the path to the Android SDK
 #  'ndk_path'    - the path to the Android NDK
 #  'javacc_path' - the path to javacc
-#  'refresh'     - True to refresh from the remote repo
 # Returns the root directory, which may be the same as 'build_dir' or may
 # be a subdirectory of it.
-def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, refresh):
+def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path):
 
     # Optionally, the actual app source can be in a subdirectory...
     if build.has_key('subdir'):

makebuildserver.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+rm -f buildserver.box
+cd buildserver
+vagrant up
+vagrant ssh -c "sudo shutdown -h now"
+cd ..
+# Just to wait until it's shut down!
+sleep 20
+vagrant package --base `VBoxManage list vms | grep buildserver | sed 's/"\(.*\)".*/\1/'` --output buildserver.box
+vagrant box add buildserver buildserver.box -f && rm buildserver.box
+

scanner.py

@@ -75,9 +75,6 @@ for app in apps:
             # Set up vcs interface and make sure we have the latest code...
             vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
 
-            refreshed_source = False
-
-
             for thisbuild in app['builds']:
 
                 if thisbuild['commit'].startswith('!'):
@@ -88,9 +85,7 @@ for app in apps:
 
                     # Prepare the source code...
                     root_dir = common.prepare_source(vcs, app, thisbuild,
-                            build_dir, extlib_dir, sdk_path, ndk_path, javacc_path,
+                            build_dir, extlib_dir, sdk_path, ndk_path, javacc_path)
-                            not refreshed_source)
-                    refreshed_source = True
 
                     # Do the scan...
                     buildprobs = common.scan_source(build_dir, root_dir, thisbuild)