Much better build server provisioning

Including selection of correct ndk architecture, caching and performance
improvements, better configurability, etc.

buildserver/.gitignore

@@ -1,2 +1,4 @@
 .vagrant
 up.log
+cache/
+Vagrantfile

buildserver/Vagrantfile

@@ -1,26 +0,0 @@
-Vagrant::Config.run do |config|
-
-  config.vm.box = "raring64"
-  config.vm.box_url = "/shares/software/OS and Boot/raring64.box"
-
-  config.vm.customize ["modifyvm", :id, "--memory", "3584"]
-
-  config.vm.provision :shell, :path => "fixpaths.sh"
-  # Set apt proxy - remove, or adjust this, accordingly!
-  config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://192.168.0.19:8000\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
-
-  config.vm.provision :chef_solo do |chef|
-    chef.cookbooks_path = "cookbooks"
-    chef.log_level = :debug 
-    chef.json = {
-      :settings => {
-        :sdk_loc => "/home/vagrant/android-sdk",
-        :ndk_loc => "/home/vagrant/android-ndk",
-        :user => "vagrant"
-      }
-    }
-    chef.add_recipe "fdroidbuild-general"
-    chef.add_recipe "android-sdk"
-    chef.add_recipe "android-ndk"
-  end
-end

buildserver/cookbooks/android-ndk/recipes/default.rb

@@ -7,7 +7,12 @@ script "setup-android-ndk" do
   user node[:settings][:user]
   cwd "/tmp"
   code "
-    tar jxvf /vagrant/cache/android-ndk-r8e-linux-x64.tar.bz2
+    if [ `uname -m` == 'x86_64' ] ; then
+       SUFFIX = '_64'
+    else
+       SUFFIX = ''
+    fi
+    tar jxvf /vagrant/cache/android-ndk-r8e-linux-x86$SUFFIX.tar.bz2
     mv android-ndk-r8e #{ndk_loc}
   "
   not_if do

buildserver/cookbooks/android-sdk/recipes/default.rb

@@ -37,10 +37,16 @@ end
     user user
     cwd "/tmp"
     code "
-      #{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
+      if [ -f /vagrant/cache/platforms/#{sdk}.tar.gz ] ; then
+        echo Installing from cache
+        tar -C #{sdk_loc}/platforms -z -x -f /vagrant/cache/platforms/#{sdk}.tar.gz
+      else
+        echo Installing via 'android'
+        #{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
 y
 
 X
+      fi
     "
     not_if "test -d #{sdk_loc}/platforms/#{sdk}"
   end
@@ -54,10 +60,16 @@ end
     user user
     cwd "/tmp"
     code "
-       #{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
+      if [ -f /vagrant/cache/add-ons/#{sdk}.tar.gz ] ; then
+        echo Installing from cache
+        tar -C #{sdk_loc}/add-ons -z -x -f /vagrant/cache/add-ons/#{sdk}.tar.gz
+      else
+        echo Installing via 'android'
+        #{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
 y
 
 X
+      fi
     "
 
     not_if "test -d #{sdk_loc}/add-ons/#{sdk}"

makebuildserver.py

@@ -5,11 +5,40 @@ import sys
 import subprocess
 import time
 
-def vagrant(params, cwd=None):
+
+# Settings, which need to be moved elsewhere ultimately...
+basebox = "raring32"
+baseboxurl = "/shares/software/OS and Boot/raring32.box"
+memory = 3584
+aptproxy = "http://192.168.0.19:8000"
+arch64 = False
+
+
+
+def vagrant(params, cwd=None, printout=False):
+    """Run vagrant.
+
+    :param: list of parameters to pass to vagrant
+    :cwd: directory to run in, or None for current directory
+    :printout: True to print output in realtime, False to just
+               return it
+    :returns: (ret, out) where ret is the return code, and out
+               is the stdout (and stderr) from vagrant
+    """
     p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    out, err = p.communicate()
-    return (p.returncode, out, err)
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    out = ''
+    if printout:
+        while True:
+            line = p.stdout.readline()
+            if len(line) == 0:
+                break
+            print line,
+            out += line
+        p.wait()
+    else:
+        out = p.communicate()[0]
+    return (p.returncode, out)
 
 boxfile = 'buildserver.box'
 serverdir = 'buildserver'
@@ -28,35 +57,84 @@ if not os.path.exists(cachedir):
     os.mkdir(cachedir)
 cachefiles = [
     ('android-sdk_r21.0.1-linux.tgz',
-     'http://dl.google.com/android/android-sdk_r21.0.1-linux.tgz',
-     'cookbooks/recipes/android-sdk/default.rb'),
-    ('android-ndk-r8e-linux-x64.tar.bz2',
-     'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x64.tar.bz2',
-     'cookbooks/recipes/android-ndk/default.rb')
-    ]
+     'http://dl.google.com/android/android-sdk_r21.0.1-linux.tgz')]
+if arch64:
+    cachefiles.extend([
+    ('android-ndk-r8e-linux-x86_64.tar.bz2',
+     'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x86_64.tar.bz2')])
+else:
+    cachefiles.extend([
+    ('android-ndk-r8e-linux-x86.tar.bz2',
+     'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x86.tar.bz2')])
 wanted = []
-for f, src, check in cachefiles:
-    if subprocess.call('grep ' + f + ' ' + check) != 0:
-        print "Cache mismatch - " + f + " is not mentioned in " + check
-        sys.exit(1)
+for f, src in cachefiles:
     if not os.path.exists(os.path.join(cachedir, f)):
         print "Downloading " + f + " to cache"
-        if subprocess.call('wget ' + src, cwd=cachedir) != 0:
+        if subprocess.call(['wget', src], cwd=cachedir) != 0:
             print "...download of " + f + " failed."
             sys.exit(1)
     wanted.append(f)
-for f in os.listdir(cachedir):
-    if not f in wanted:
-        print "Removing unwanted cache file " + f
-        os.remove(os.path.join(cachedir, f))
 
 
-vagrant(['halt'], serverdir)
+# Generate an appropriate Vagrantfile for the buildserver, based on our
+# settings...
+vagrantfile = """
+Vagrant::Config.run do |config|
+
+  config.vm.box = "{0}"
+  config.vm.box_url = "{1}"
+
+  config.vm.customize ["modifyvm", :id, "--memory", "{2}"]
+
+  config.vm.provision :shell, :path => "fixpaths.sh"
+""".format(basebox, baseboxurl, memory)
+if aptproxy:
+    vagrantfile += """
+  config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
+""".format(aptproxy)
+vagrantfile += """
+  config.vm.provision :chef_solo do |chef|
+    chef.cookbooks_path = "cookbooks"
+    chef.log_level = :debug 
+    chef.json = {
+      :settings => {
+        :sdk_loc => "/home/vagrant/android-sdk",
+        :ndk_loc => "/home/vagrant/android-ndk",
+        :user => "vagrant"
+      }
+    }
+    chef.add_recipe "fdroidbuild-general"
+    chef.add_recipe "android-sdk"
+    chef.add_recipe "android-ndk"
+  end
+end
+"""
+
+# Check against the existing Vagrantfile, and if they differ, we need to
+# create a new box:
+vf = os.path.join(serverdir, 'Vagrantfile')
+writevf = True
+if os.path.exists(vf):
+    vagrant(['halt'], serverdir)
+    with open(vf, 'r') as f:
+        oldvf = f.read()
+    if oldvf != vagrantfile:
+        print "Server configuration has changed, rebuild from scratch is required"
+        vagrant(['destroy', '-f'], serverdir)
+    else:
+        print "Re-provisioning existing server"
+        writevf = False
+else:
+    print "No existing server - building from scratch"
+if writevf:
+    with open(vf, 'w') as f:
+        f.write(vagrantfile)
+
+
 print "Configuring build server VM"
-returncode, out, err = vagrant(['up'], serverdir)
+returncode, out = vagrant(['up'], serverdir, printout=True)
 with open(os.path.join(serverdir, 'up.log'), 'w') as log:
-    log.write('==stdout==\n' + out + '\n\n')
-    log.write('==stderr==\n' + err + '\n\n')
+    log.write(out)
 if returncode != 0:
     print "Failed to configure server"
     sys.exit(1)
@@ -67,7 +145,7 @@ print "Waiting for build server VM to be finished"
 ready = False
 while not ready:
     time.sleep(2)
-    returncode, out, err = vagrant(['status'], serverdir)
+    returncode, out = vagrant(['status'], serverdir)
     if returncode != 0:
         print "Error while checking status"
         sys.exit(1)