diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 51db27d3..c353b506 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -15,10 +15,28 @@ Vagrant.configure("2") do |config| config.vm.box = configfile['basebox'] config.vm.box_url = configfile['baseboxurl'] - config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--memory", configfile['memory']] - v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] - v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] + # TODO detect if jessie64.box is libvirt, or `vagrant mutate jessie64 libvirt` + if `systemd-detect-virt`.include? "qemu" or configfile["vm_provider"] == "libvirt" + # use KVM/QEMU if this is running in KVM/QEMU + config.vm.provider :libvirt do |libvirt| + libvirt.driver = "kvm" + libvirt.host = "localhost" + libvirt.uri = "qemu:///system" + libvirt.cpus = configfile["cpus"] + libvirt.memory = configfile["memory"] + end + config.vm.synced_folder './', '/vagrant', type: '9p' + synced_folder_type = '9p' + elsif not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox" + # default to VirtualBox if not set + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--memory", configfile['memory']] + v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] + v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] + end + synced_folder_type = 'virtualbox' + else + abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!") end config.vm.boot_timeout = configfile['boot_timeout'] @@ -32,8 +50,12 @@ Vagrant.configure("2") do |config| # default does not need a custom mount if configfile["cachedir"] != "buildserver/cache" config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', - owner: 'root', group: 'root', create: true + create: true, type: synced_folder_type end + # Make sure dir exists to mount to, since buildserver/ is + # automatically mounted as /vagrant in the guest VM. This is more + # necessary with 9p synced folders + Dir.mkdir('cache') unless File.exists?('cache') # cache .deb packages on the host via a mount trick if configfile.has_key? "aptcachedir" diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 57eacdc4..7c9b3286 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -23,10 +23,27 @@ if [ ! -x $ANDROID_HOME/tools/android ]; then unzip -qq $tools fi +# disable the repositories of proprietary stuff +disabled=" +@version@=1 +@disabled@https\://dl.google.com/android/repository/extras/intel/addon.xml=disabled +@disabled@https\://dl.google.com/android/repository/glass/addon.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/android/sys-img.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/android-tv/sys-img.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/android-wear/sys-img.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/google_apis/sys-img.xml=disabled +" +test -d ${HOME}/.android || mkdir ${HOME}/.android +for line in $disabled; do + echo $line >> ${HOME}/.android/sites-settings.cfg +done + + cd /vagrant/cache # make links for `android update sdk` to use and delete -for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do +latestm2=`ls -1 android_m2repository*.zip | sort -n | tail -1` +for f in $latestm2 android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do rm -f ${ANDROID_HOME}/temp/$f ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/ done diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 09909050..e702f874 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -11,6 +11,9 @@ sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources. printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \ > /etc/apt/apt.conf.d/99no-install-recommends +printf 'APT::Acquire::Retries "20";\n' \ + > /etc/apt/apt.conf.d/99acquire-retries + if grep --quiet jessie /etc/apt/sources.list; then echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list fi diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 4ee73fbc..b6c0183d 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -48,3 +48,11 @@ # about the timeout, extend the timeout here. (default: 600 seconds) # # boot_timeout = 1200 + +# By default, this whole process uses VirtualBox as the provider, but +# QEMU+KVM is also supported via the libvirt plugin to vagrant. If +# this is run within a KVM guest, then libvirt's QEMU+KVM will be used +# automatically. It can also be manually enabled by uncommenting +# below: +# +# vm_provider = 'libvirt' diff --git a/examples/public-read-only-s3-bucket-policy.json b/examples/public-read-only-s3-bucket-policy.json new file mode 100644 index 00000000..9316bbe0 --- /dev/null +++ b/examples/public-read-only-s3-bucket-policy.json @@ -0,0 +1,11 @@ +{ + "Version":"2012-10-17", + "Statement":[ + {"Sid":"AddPerm", + "Effect":"Allow", + "Principal":"*", + "Action":"s3:GetObject", + "Resource":"arn:aws:s3:::examplebucket/fdroid/*" + } + ] +} diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index b78c4adf..d77c241c 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -1,3 +1,4 @@ + #!/bin/bash if [ `dirname $0` != "." ]; then @@ -11,14 +12,15 @@ fi # make sure that no VirtualBox processes are left running cleanup_all() { + set +e echo "$(date -u) - cleanup in progress..." - ps auxww|grep VBox + ps auxww | grep -e VBox -e qemu cd $WORKSPACE/buildserver - vagrant halt || true + vagrant halt sleep 5 - killall VBoxHeadless || true + killall VBoxHeadless sleep 5 - killall -9 VBoxHeadless || true + killall -9 VBoxHeadless echo "$(date -u) - cleanup done." } trap cleanup_all INT TERM EXIT @@ -37,8 +39,10 @@ sha256sum -c jessie64.box.sha256 export XDG_CONFIG_HOME=$WORKSPACE export VBOX_USER_HOME=$WORKSPACE/VirtualBox mkdir $VBOX_USER_HOME -VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d -VBoxManage setproperty logginglevel debug +if which VBoxManage; then + VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d + VBoxManage setproperty logginglevel debug +fi export VAGRANT_HOME=$WORKSPACE/vagrant.d mkdir $VAGRANT_HOME @@ -46,7 +50,7 @@ cd $WORKSPACE echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py -./makebuildserver --verbose +./makebuildserver --verbose --clean # this can be handled in the jenkins job, or here: if [ -e fdroiddata ]; then diff --git a/makebuildserver b/makebuildserver index e6c5b84b..0f3123ba 100755 --- a/makebuildserver +++ b/makebuildserver @@ -2,6 +2,7 @@ import os import requests +import stat import sys import subprocess import time @@ -63,8 +64,20 @@ config = { 'cpus': 1, 'memory': 1024, 'hwvirtex': 'off', + 'vm_provider': 'virtualbox', } +if os.path.isfile('/usr/bin/systemd-detect-virt'): + try: + virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8') + except subprocess.CalledProcessError as e: + virt = 'none' + if virt == 'qemu' or virt == 'kvm': + print('Running in a VM guest, defaulting to QEMU/KVM via libvirt') + config['vm_provider'] = 'libvirt' + elif virt != 'none': + print('Running in an unsupported VM guest (' + virt + ')!') + # load config file, if present if os.path.exists('makebuildserver.config.py'): exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config) @@ -89,6 +102,15 @@ cachedir = config['cachedir'] if not os.path.exists(cachedir): os.makedirs(cachedir, 0o755) +tmp = cachedir +while tmp != '/': + mode = os.stat(tmp).st_mode + if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode): + print('ERROR:', tmp, 'will not be accessible to the VM! To fix, run:') + print(' chmod a+X', tmp) + sys.exit(1) + tmp = os.path.dirname(tmp) + if config['apt_package_cache']: config['aptcachedir'] = cachedir + '/apt/archives' @@ -338,6 +360,7 @@ elif os.path.exists('/proc/cpuinfo'): vf = os.path.join(serverdir, 'Vagrantfile.yaml') writevf = True if os.path.exists(vf): + print('Halting', serverdir) vagrant(['halt'], serverdir) with open(vf, 'r', encoding='utf-8') as f: oldconfig = yaml.load(f) @@ -353,6 +376,33 @@ if writevf: with open(vf, 'w', encoding='utf-8') as f: yaml.dump(config, f) +if config['vm_provider'] == 'libvirt': + returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose) + found_basebox = False + needs_mutate = False + for line in out.splitlines(): + if line.startswith(config['basebox']): + found_basebox = True + if line.split('(')[1].split(',')[0] != 'libvirt': + needs_mutate = True + continue + if not found_basebox: + if isinstance(config['baseboxurl'], str): + baseboxurl = config['baseboxurl'] + else: + baseboxurl = config['baseboxurl'][0] + print('Adding', config['basebox'], 'from', baseboxurl) + vagrant(['box', 'add', '--name', config['basebox'], baseboxurl], + serverdir, printout=options.verbose) + needs_mutate = True + if needs_mutate: + print('Converting', config['basebox'], 'to libvirt format') + vagrant(['mutate', config['basebox'], 'libvirt'], + serverdir, printout=options.verbose) + print('Removing virtualbox format copy of', config['basebox']) + vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']], + serverdir, printout=options.verbose) + print("Configuring build server VM") returncode, out = vagrant(['up', '--provision'], serverdir, printout=True) with open(os.path.join(serverdir, 'up.log'), 'w') as log: diff --git a/tests/import.TestCase b/tests/import.TestCase index 4d5acc56..7b846a5f 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -5,6 +5,7 @@ import inspect import optparse import os +import requests import sys import unittest @@ -30,6 +31,12 @@ class ImportTest(unittest.TestCase): fdroidserver.common.config = config url = 'https://gitlab.com/fdroid/fdroidclient' + r = requests.head(url) + if r.status_code != 200: + print("ERROR", url, 'unreachable (', r.status_code, ')') + print('Skipping ImportTest!') + return + app = fdroidserver.metadata.get_default_app_info() app.UpdateCheckMode = "Tags" root_dir, src_dir = import_proxy.get_metadata_from_url(app, url)