From 008b4a31ebd169186329031e0fec4dab52703ff6 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Sep 2016 08:29:18 +0200 Subject: [PATCH 01/10] skip `fdroid import` test if gitlab is not available This prevents CI build failures when gitlab is deploying or has other issues that might cause a 500. --- tests/import.TestCase | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/import.TestCase b/tests/import.TestCase index 4d5acc56..7b846a5f 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -5,6 +5,7 @@ import inspect import optparse import os +import requests import sys import unittest @@ -30,6 +31,12 @@ class ImportTest(unittest.TestCase): fdroidserver.common.config = config url = 'https://gitlab.com/fdroid/fdroidclient' + r = requests.head(url) + if r.status_code != 200: + print("ERROR", url, 'unreachable (', r.status_code, ')') + print('Skipping ImportTest!') + return + app = fdroidserver.metadata.get_default_app_info() app.UpdateCheckMode = "Tags" root_dir, src_dir = import_proxy.get_metadata_from_url(app, url) From b5e57a7aa67033f1ba0bdb351d0563a384dd3d63 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Sep 2016 11:35:38 +0200 Subject: [PATCH 02/10] example file for public read only S3 bucket policy This is the locked down S3 policy as created by Benetech for their Secure App Generator project. --- examples/public-read-only-s3-bucket-policy.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 examples/public-read-only-s3-bucket-policy.json diff --git a/examples/public-read-only-s3-bucket-policy.json b/examples/public-read-only-s3-bucket-policy.json new file mode 100644 index 00000000..9316bbe0 --- /dev/null +++ b/examples/public-read-only-s3-bucket-policy.json @@ -0,0 +1,11 @@ +{ + "Version":"2012-10-17", + "Statement":[ + {"Sid":"AddPerm", + "Effect":"Allow", + "Principal":"*", + "Action":"s3:GetObject", + "Resource":"arn:aws:s3:::examplebucket/fdroid/*" + } + ] +} From 5af53238d398c41e09f9f0c6e3923025a5702028 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Sep 2016 11:29:31 +0200 Subject: [PATCH 03/10] buildserver: ignore android sdk repos of proprietary bits The Android SDK by default includes some Google repositories of their proprietary SDKs. We of course do not want that stuff ever. We also do not need the emulator images since this process does not currently install or run an emulator. --- buildserver/provision-android-sdk | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 57eacdc4..946f1e70 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -23,6 +23,22 @@ if [ ! -x $ANDROID_HOME/tools/android ]; then unzip -qq $tools fi +# disable the repositories of proprietary stuff +disabled=" +@version@=1 +@disabled@https\://dl.google.com/android/repository/extras/intel/addon.xml=disabled +@disabled@https\://dl.google.com/android/repository/glass/addon.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/android/sys-img.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/android-tv/sys-img.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/android-wear/sys-img.xml=disabled +@disabled@https\://dl.google.com/android/repository/sys-img/google_apis/sys-img.xml=disabled +" +test -d ${HOME}/.android || mkdir ${HOME}/.android +for line in $disabled; do + echo $line >> ${HOME}/.android/sites-settings.cfg +done + + cd /vagrant/cache # make links for `android update sdk` to use and delete From 0ae2df6646dfe927475378eba2b762a3ec86b0e7 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 5 Sep 2016 23:21:24 +0200 Subject: [PATCH 04/10] buildserver: support optionally using qemu+kvm This makes it possible to run the full build process in a KVM virtual machine, like jenkins.debian.net. closes #190 --- buildserver/Vagrantfile | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 51db27d3..c182e07e 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -15,10 +15,28 @@ Vagrant.configure("2") do |config| config.vm.box = configfile['basebox'] config.vm.box_url = configfile['baseboxurl'] - config.vm.provider "virtualbox" do |v| - v.customize ["modifyvm", :id, "--memory", configfile['memory']] - v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] - v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] + # TODO detect if jessie64.box is libvirt, or `vagrant mutate jessie64 libvirt` + if `systemd-detect-virt`.include? "qemu" or configfile["vm_provider"] == "libvirt" + # use KVM/QEMU if this is running in KVM/QEMU + config.vm.provider :libvirt do |libvirt| + libvirt.driver = "kvm" + libvirt.host = "localhost" + libvirt.uri = "qemu:///system" + libvirt.cpus = configfile["cpus"] + libvirt.memory = configfile["memory"] + end + config.vm.synced_folder './', '/vagrant', type: '9p' + synced_folder_type = '9p' + elsif not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox" + # default to VirtualBox if not set + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--memory", configfile['memory']] + v.customize ["modifyvm", :id, "--cpus", configfile['cpus']] + v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']] + end + synced_folder_type = 'virtualbox' + else + abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!") end config.vm.boot_timeout = configfile['boot_timeout'] @@ -32,7 +50,7 @@ Vagrant.configure("2") do |config| # default does not need a custom mount if configfile["cachedir"] != "buildserver/cache" config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', - owner: 'root', group: 'root', create: true + create: true, type: synced_folder_type end # cache .deb packages on the host via a mount trick From f306e32636fc4aa8cf2d6170c1c415bc4dc9a7cc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 6 Sep 2016 14:22:29 +0200 Subject: [PATCH 05/10] buildserver: make sure cachedir is accessible to VM instance It seems that KVM/QEMU's 9p sharing is more sensitive to file perms. If ~/.cache is 0700, then ~/.cache/fdroidserver cannot be mounted in the guest --- makebuildserver | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/makebuildserver b/makebuildserver index 365c03da..3a26bf51 100755 --- a/makebuildserver +++ b/makebuildserver @@ -2,6 +2,7 @@ import os import requests +import stat import sys import subprocess import time @@ -89,6 +90,15 @@ cachedir = config['cachedir'] if not os.path.exists(cachedir): os.makedirs(cachedir, 0o755) +tmp = cachedir +while tmp != '/': + mode = os.stat(tmp).st_mode + if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode): + print('ERROR:', tmp, 'will not be accessible to the VM! To fix, run:') + print(' chmod a+X', tmp) + sys.exit(1) + tmp = os.path.dirname(tmp) + if config['apt_package_cache']: config['aptcachedir'] = cachedir + '/apt/archives' From 68ec7c2e88641e1739d8e5eda4588938b915b1b9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Sep 2016 11:12:18 +0200 Subject: [PATCH 06/10] buildserver: auto-detect and use libvirt's QEMU+KVM For running in QEMU/KVM guests like on jenkins.debian.net, this sets up the whole process automatically. This only really covers the case where this is running in a KVM guest, and the original case of running VirtualBox on bare metal. It could be extended to cover more cases if someone wanted to. --- examples/makebuildserver.config.py | 8 ++++++ makebuildserver | 40 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index 4ee73fbc..b6c0183d 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -48,3 +48,11 @@ # about the timeout, extend the timeout here. (default: 600 seconds) # # boot_timeout = 1200 + +# By default, this whole process uses VirtualBox as the provider, but +# QEMU+KVM is also supported via the libvirt plugin to vagrant. If +# this is run within a KVM guest, then libvirt's QEMU+KVM will be used +# automatically. It can also be manually enabled by uncommenting +# below: +# +# vm_provider = 'libvirt' diff --git a/makebuildserver b/makebuildserver index 3a26bf51..19010137 100755 --- a/makebuildserver +++ b/makebuildserver @@ -64,8 +64,20 @@ config = { 'cpus': 1, 'memory': 1024, 'hwvirtex': 'off', + 'vm_provider': 'virtualbox', } +if os.path.isfile('/usr/bin/systemd-detect-virt'): + try: + virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8') + except subprocess.CalledProcessError as e: + virt = 'none' + if virt == 'qemu' or virt == 'kvm': + print('Running in a VM guest, defaulting to QEMU/KVM via libvirt') + config['vm_provider'] = 'libvirt' + elif virt != 'none': + print('Running in an unsupported VM guest (' + virt + ')!') + # load config file, if present if os.path.exists('makebuildserver.config.py'): exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config) @@ -346,6 +358,7 @@ elif os.path.exists('/proc/cpuinfo'): vf = os.path.join(serverdir, 'Vagrantfile.yaml') writevf = True if os.path.exists(vf): + print('Halting', serverdir) vagrant(['halt'], serverdir) with open(vf, 'r', encoding='utf-8') as f: oldconfig = yaml.load(f) @@ -361,6 +374,33 @@ if writevf: with open(vf, 'w', encoding='utf-8') as f: yaml.dump(config, f) +if config['vm_provider'] == 'libvirt': + returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose) + found_basebox = False + needs_mutate = False + for line in out.splitlines(): + if line.startswith(config['basebox']): + found_basebox = True + if line.split('(')[1].split(',')[0] != 'libvirt': + needs_mutate = True + continue + if not found_basebox: + if isinstance(config['baseboxurl'], str): + baseboxurl = config['baseboxurl'] + else: + baseboxurl = config['baseboxurl'][0] + print('Adding', config['basebox'], 'from', baseboxurl) + vagrant(['box', 'add', '--name', config['basebox'], baseboxurl], + serverdir, printout=options.verbose) + needs_mutate = True + if needs_mutate: + print('Converting', config['basebox'], 'to libvirt format') + vagrant(['mutate', config['basebox'], 'libvirt'], + serverdir, printout=options.verbose) + print('Removing virtualbox format copy of', config['basebox']) + vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']], + serverdir, printout=options.verbose) + print("Configuring build server VM") returncode, out = vagrant(['up', '--provision'], serverdir, printout=True) with open(os.path.join(serverdir, 'up.log'), 'w') as log: From 3651e7eb9aaefecd03be67ff4d6d858273f5c56e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Sep 2016 11:12:49 +0200 Subject: [PATCH 07/10] buildserver: retry apt-get downloads 20 times Try harder before failing the whole buildserver setup. --- buildserver/provision-apt-get-install | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 09909050..e702f874 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -11,6 +11,9 @@ sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources. printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \ > /etc/apt/apt.conf.d/99no-install-recommends +printf 'APT::Acquire::Retries "20";\n' \ + > /etc/apt/apt.conf.d/99acquire-retries + if grep --quiet jessie /etc/apt/sources.list; then echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list fi From 988647a3871a81b88014b7742f471c8f47e48fa8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Sep 2016 14:38:36 +0200 Subject: [PATCH 08/10] buildserver: fix jenkins build to support QEMU/KVM --- jenkins-build-makebuildserver | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index b78c4adf..d77c241c 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -1,3 +1,4 @@ + #!/bin/bash if [ `dirname $0` != "." ]; then @@ -11,14 +12,15 @@ fi # make sure that no VirtualBox processes are left running cleanup_all() { + set +e echo "$(date -u) - cleanup in progress..." - ps auxww|grep VBox + ps auxww | grep -e VBox -e qemu cd $WORKSPACE/buildserver - vagrant halt || true + vagrant halt sleep 5 - killall VBoxHeadless || true + killall VBoxHeadless sleep 5 - killall -9 VBoxHeadless || true + killall -9 VBoxHeadless echo "$(date -u) - cleanup done." } trap cleanup_all INT TERM EXIT @@ -37,8 +39,10 @@ sha256sum -c jessie64.box.sha256 export XDG_CONFIG_HOME=$WORKSPACE export VBOX_USER_HOME=$WORKSPACE/VirtualBox mkdir $VBOX_USER_HOME -VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d -VBoxManage setproperty logginglevel debug +if which VBoxManage; then + VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d + VBoxManage setproperty logginglevel debug +fi export VAGRANT_HOME=$WORKSPACE/vagrant.d mkdir $VAGRANT_HOME @@ -46,7 +50,7 @@ cd $WORKSPACE echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py -./makebuildserver --verbose +./makebuildserver --verbose --clean # this can be handled in the jenkins job, or here: if [ -e fdroiddata ]; then From 0a0ba6de84485ae8504a7cef48c804bfbea2e393 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 15 Sep 2016 15:50:16 +0200 Subject: [PATCH 09/10] buildserver: ensure dirs to mount cache exist in guest VM instance It seems that the 9p synced folder setup is not as flexible and easy as the VirtualBox one, so we have to do more little things like this. --- buildserver/Vagrantfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index c182e07e..c353b506 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -52,6 +52,10 @@ Vagrant.configure("2") do |config| config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', create: true, type: synced_folder_type end + # Make sure dir exists to mount to, since buildserver/ is + # automatically mounted as /vagrant in the guest VM. This is more + # necessary with 9p synced folders + Dir.mkdir('cache') unless File.exists?('cache') # cache .deb packages on the host via a mount trick if configfile.has_key? "aptcachedir" From a2a630c83c87285ba95ada59810e5aae4a8dcb03 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 19 Sep 2016 13:29:46 +0200 Subject: [PATCH 10/10] buildserver: only include latest m2 when provisioning I think the `android update sdk` tool is installing all of the m2 files that are present in the temp cache, and it seems to do it in order of newest to oldest. Well done, and I thought that tool couldn't get any worse. So only include the latest version of android_m2repository*.zip in the temp cache. --- buildserver/provision-android-sdk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index 946f1e70..7c9b3286 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -42,7 +42,8 @@ done cd /vagrant/cache # make links for `android update sdk` to use and delete -for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do +latestm2=`ls -1 android_m2repository*.zip | sort -n | tail -1` +for f in $latestm2 android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do rm -f ${ANDROID_HOME}/temp/$f ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/ done