Merge branch 'buildserver-qemu-kvm' into 'master'

buildserver running in qemu/kvm to support KVM on KVM

jenkins.debian.net runs in QEMU/KVM instances, so in order to run the F-Droid buildserver there, it needs to work inside of a KVM guest.  The best way I found to do that is to create QEMU/KVM instances via KVM's "nested" virtualization support.  This collection of commits enables using QEMU/KVM as the buildserver when `./makebuildserver` detects that it is running inside of KVM.  Otherwise, the old behavior is default: running in VirtualBox.

I have run these tests inside of ubuntu/16.04 on bare metal, which uses VirtualBox, and ubuntu/16.04 KVM guest, which uses QEMU/KVM.  It'll also run on the Guardian Project jenkins box, which is Debian/jessie.

@mvdan @CiaranG @krt

See merge request !168

buildserver/Vagrantfile

@@ -15,10 +15,28 @@ Vagrant.configure("2") do |config|
   config.vm.box = configfile['basebox']
   config.vm.box_url = configfile['baseboxurl']
 
-  config.vm.provider "virtualbox" do |v|
-    v.customize ["modifyvm", :id, "--memory", configfile['memory']]
-    v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
-    v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
+  # TODO detect if jessie64.box is libvirt, or `vagrant mutate jessie64 libvirt`
+  if `systemd-detect-virt`.include? "qemu" or configfile["vm_provider"] == "libvirt"
+    # use KVM/QEMU if this is running in KVM/QEMU
+    config.vm.provider :libvirt do |libvirt|
+      libvirt.driver = "kvm"
+      libvirt.host = "localhost"
+      libvirt.uri = "qemu:///system"
+      libvirt.cpus = configfile["cpus"]
+      libvirt.memory = configfile["memory"]
+    end
+    config.vm.synced_folder './', '/vagrant', type: '9p'
+    synced_folder_type = '9p'
+  elsif not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox"
+    # default to VirtualBox if not set
+    config.vm.provider "virtualbox" do |v|
+      v.customize ["modifyvm", :id, "--memory", configfile['memory']]
+      v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
+      v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
+    end
+    synced_folder_type = 'virtualbox'
+  else
+    abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!")
   end
 
   config.vm.boot_timeout = configfile['boot_timeout']
@@ -32,8 +50,12 @@ Vagrant.configure("2") do |config|
   # default does not need a custom mount
   if configfile["cachedir"] != "buildserver/cache"
     config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
-      owner: 'root', group: 'root', create: true
+      create: true, type: synced_folder_type
   end
+  # Make sure dir exists to mount to, since buildserver/ is
+  # automatically mounted as /vagrant in the guest VM. This is more
+  # necessary with 9p synced folders
+  Dir.mkdir('cache') unless File.exists?('cache')
 
   # cache .deb packages on the host via a mount trick
   if configfile.has_key? "aptcachedir"

buildserver/provision-android-sdk

@@ -23,10 +23,27 @@ if [ ! -x $ANDROID_HOME/tools/android ]; then
     unzip -qq $tools
 fi
 
+# disable the repositories of proprietary stuff
+disabled="
+@version@=1
+@disabled@https\://dl.google.com/android/repository/extras/intel/addon.xml=disabled
+@disabled@https\://dl.google.com/android/repository/glass/addon.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android-tv/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/android-wear/sys-img.xml=disabled
+@disabled@https\://dl.google.com/android/repository/sys-img/google_apis/sys-img.xml=disabled
+"
+test -d ${HOME}/.android || mkdir ${HOME}/.android
+for line in $disabled; do
+    echo $line >> ${HOME}/.android/sites-settings.cfg
+done
+
+
 cd /vagrant/cache
 
 # make links for `android update sdk` to use and delete
-for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do
+latestm2=`ls -1 android_m2repository*.zip | sort -n | tail -1`
+for f in $latestm2 android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do
     rm -f ${ANDROID_HOME}/temp/$f
     ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/
 done

buildserver/provision-apt-get-install

@@ -11,6 +11,9 @@ sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.
 printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \
        > /etc/apt/apt.conf.d/99no-install-recommends
 
+printf 'APT::Acquire::Retries "20";\n' \
+       > /etc/apt/apt.conf.d/99acquire-retries
+
 if grep --quiet jessie /etc/apt/sources.list; then
     echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list
 fi

examples/makebuildserver.config.py

@@ -48,3 +48,11 @@
 # about the timeout, extend the timeout here. (default: 600 seconds)
 #
 # boot_timeout = 1200
+
+# By default, this whole process uses VirtualBox as the provider, but
+# QEMU+KVM is also supported via the libvirt plugin to vagrant. If
+# this is run within a KVM guest, then libvirt's QEMU+KVM will be used
+# automatically.  It can also be manually enabled by uncommenting
+# below:
+#
+# vm_provider = 'libvirt'

examples/public-read-only-s3-bucket-policy.json

@@ -0,0 +1,11 @@
+{
+    "Version":"2012-10-17",
+    "Statement":[
+        {"Sid":"AddPerm",
+         "Effect":"Allow",
+         "Principal":"*",
+         "Action":"s3:GetObject",
+         "Resource":"arn:aws:s3:::examplebucket/fdroid/*"
+        }
+    ]
+}

jenkins-build-makebuildserver

@@ -1,3 +1,4 @@
+
 #!/bin/bash
 
 if [ `dirname $0` != "." ]; then
@@ -11,14 +12,15 @@ fi
 
 # make sure that no VirtualBox processes are left running
 cleanup_all() {
+    set +e
     echo "$(date -u) - cleanup in progress..."
-    ps auxww|grep VBox
+    ps auxww | grep -e VBox -e qemu
     cd $WORKSPACE/buildserver
-    vagrant halt || true
+    vagrant halt
     sleep 5
-    killall VBoxHeadless || true
+    killall VBoxHeadless
     sleep 5
-    killall -9 VBoxHeadless || true
+    killall -9 VBoxHeadless
     echo "$(date -u) - cleanup done."
 }
 trap cleanup_all INT TERM EXIT
@@ -37,8 +39,10 @@ sha256sum -c jessie64.box.sha256
 export XDG_CONFIG_HOME=$WORKSPACE
 export VBOX_USER_HOME=$WORKSPACE/VirtualBox
 mkdir $VBOX_USER_HOME
-VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d
-VBoxManage setproperty logginglevel debug
+if which VBoxManage; then
+    VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d
+    VBoxManage setproperty logginglevel debug
+fi
 export VAGRANT_HOME=$WORKSPACE/vagrant.d
 mkdir $VAGRANT_HOME
 
@@ -46,7 +50,7 @@ cd $WORKSPACE
 echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
 echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
 echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
-./makebuildserver --verbose
+./makebuildserver --verbose --clean
 
 # this can be handled in the jenkins job, or here:
 if [ -e fdroiddata ]; then

makebuildserver

@@ -2,6 +2,7 @@
 
 import os
 import requests
+import stat
 import sys
 import subprocess
 import time
@@ -63,8 +64,20 @@ config = {
     'cpus': 1,
     'memory': 1024,
     'hwvirtex': 'off',
+    'vm_provider': 'virtualbox',
 }
 
+if os.path.isfile('/usr/bin/systemd-detect-virt'):
+    try:
+        virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
+    except subprocess.CalledProcessError as e:
+        virt = 'none'
+    if virt == 'qemu' or virt == 'kvm':
+        print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
+        config['vm_provider'] = 'libvirt'
+    elif virt != 'none':
+        print('Running in an unsupported VM guest (' + virt + ')!')
+
 # load config file, if present
 if os.path.exists('makebuildserver.config.py'):
     exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
@@ -89,6 +102,15 @@ cachedir = config['cachedir']
 if not os.path.exists(cachedir):
     os.makedirs(cachedir, 0o755)
 
+tmp = cachedir
+while tmp != '/':
+    mode = os.stat(tmp).st_mode
+    if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
+        print('ERROR:', tmp, 'will not be accessible to the VM!  To fix, run:')
+        print('  chmod a+X', tmp)
+        sys.exit(1)
+    tmp = os.path.dirname(tmp)
+
 if config['apt_package_cache']:
     config['aptcachedir'] = cachedir + '/apt/archives'
 
@@ -338,6 +360,7 @@ elif os.path.exists('/proc/cpuinfo'):
 vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 writevf = True
 if os.path.exists(vf):
+    print('Halting', serverdir)
     vagrant(['halt'], serverdir)
     with open(vf, 'r', encoding='utf-8') as f:
         oldconfig = yaml.load(f)
@@ -353,6 +376,33 @@ if writevf:
     with open(vf, 'w', encoding='utf-8') as f:
         yaml.dump(config, f)
 
+if config['vm_provider'] == 'libvirt':
+    returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose)
+    found_basebox = False
+    needs_mutate = False
+    for line in out.splitlines():
+        if line.startswith(config['basebox']):
+            found_basebox = True
+            if line.split('(')[1].split(',')[0] != 'libvirt':
+                needs_mutate = True
+            continue
+    if not found_basebox:
+        if isinstance(config['baseboxurl'], str):
+            baseboxurl = config['baseboxurl']
+        else:
+            baseboxurl = config['baseboxurl'][0]
+        print('Adding', config['basebox'], 'from', baseboxurl)
+        vagrant(['box', 'add', '--name', config['basebox'], baseboxurl],
+                serverdir, printout=options.verbose)
+        needs_mutate = True
+    if needs_mutate:
+        print('Converting', config['basebox'], 'to libvirt format')
+        vagrant(['mutate', config['basebox'], 'libvirt'],
+                serverdir, printout=options.verbose)
+        print('Removing virtualbox format copy of', config['basebox'])
+        vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']],
+                serverdir, printout=options.verbose)
+
 print("Configuring build server VM")
 returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)
 with open(os.path.join(serverdir, 'up.log'), 'w') as log:

tests/import.TestCase

@@ -5,6 +5,7 @@
 import inspect
 import optparse
 import os
+import requests
 import sys
 import unittest
 
@@ -30,6 +31,12 @@ class ImportTest(unittest.TestCase):
         fdroidserver.common.config = config
 
         url = 'https://gitlab.com/fdroid/fdroidclient'
+        r = requests.head(url)
+        if r.status_code != 200:
+            print("ERROR", url, 'unreachable (', r.status_code, ')')
+            print('Skipping ImportTest!')
+            return
+
         app = fdroidserver.metadata.get_default_app_info()
         app.UpdateCheckMode = "Tags"
         root_dir, src_dir = import_proxy.get_metadata_from_url(app, url)