mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-05 18:50:09 +02:00
Merge branch 'buildserver-qemu-kvm' into 'master'
buildserver running in qemu/kvm to support KVM on KVM jenkins.debian.net runs in QEMU/KVM instances, so in order to run the F-Droid buildserver there, it needs to work inside of a KVM guest. The best way I found to do that is to create QEMU/KVM instances via KVM's "nested" virtualization support. This collection of commits enables using QEMU/KVM as the buildserver when `./makebuildserver` detects that it is running inside of KVM. Otherwise, the old behavior is default: running in VirtualBox. I have run these tests inside of ubuntu/16.04 on bare metal, which uses VirtualBox, and ubuntu/16.04 KVM guest, which uses QEMU/KVM. It'll also run on the Guardian Project jenkins box, which is Debian/jessie. @mvdan @CiaranG @krt See merge request !168
This commit is contained in:
commit
5667d16498
32
buildserver/Vagrantfile
vendored
32
buildserver/Vagrantfile
vendored
@ -15,10 +15,28 @@ Vagrant.configure("2") do |config|
|
|||||||
config.vm.box = configfile['basebox']
|
config.vm.box = configfile['basebox']
|
||||||
config.vm.box_url = configfile['baseboxurl']
|
config.vm.box_url = configfile['baseboxurl']
|
||||||
|
|
||||||
config.vm.provider "virtualbox" do |v|
|
# TODO detect if jessie64.box is libvirt, or `vagrant mutate jessie64 libvirt`
|
||||||
v.customize ["modifyvm", :id, "--memory", configfile['memory']]
|
if `systemd-detect-virt`.include? "qemu" or configfile["vm_provider"] == "libvirt"
|
||||||
v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
|
# use KVM/QEMU if this is running in KVM/QEMU
|
||||||
v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
|
config.vm.provider :libvirt do |libvirt|
|
||||||
|
libvirt.driver = "kvm"
|
||||||
|
libvirt.host = "localhost"
|
||||||
|
libvirt.uri = "qemu:///system"
|
||||||
|
libvirt.cpus = configfile["cpus"]
|
||||||
|
libvirt.memory = configfile["memory"]
|
||||||
|
end
|
||||||
|
config.vm.synced_folder './', '/vagrant', type: '9p'
|
||||||
|
synced_folder_type = '9p'
|
||||||
|
elsif not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox"
|
||||||
|
# default to VirtualBox if not set
|
||||||
|
config.vm.provider "virtualbox" do |v|
|
||||||
|
v.customize ["modifyvm", :id, "--memory", configfile['memory']]
|
||||||
|
v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
|
||||||
|
v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
|
||||||
|
end
|
||||||
|
synced_folder_type = 'virtualbox'
|
||||||
|
else
|
||||||
|
abort("No supported VM Provider found, set vm_provider in Vagrantfile.yaml!")
|
||||||
end
|
end
|
||||||
|
|
||||||
config.vm.boot_timeout = configfile['boot_timeout']
|
config.vm.boot_timeout = configfile['boot_timeout']
|
||||||
@ -32,8 +50,12 @@ Vagrant.configure("2") do |config|
|
|||||||
# default does not need a custom mount
|
# default does not need a custom mount
|
||||||
if configfile["cachedir"] != "buildserver/cache"
|
if configfile["cachedir"] != "buildserver/cache"
|
||||||
config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
|
config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
|
||||||
owner: 'root', group: 'root', create: true
|
create: true, type: synced_folder_type
|
||||||
end
|
end
|
||||||
|
# Make sure dir exists to mount to, since buildserver/ is
|
||||||
|
# automatically mounted as /vagrant in the guest VM. This is more
|
||||||
|
# necessary with 9p synced folders
|
||||||
|
Dir.mkdir('cache') unless File.exists?('cache')
|
||||||
|
|
||||||
# cache .deb packages on the host via a mount trick
|
# cache .deb packages on the host via a mount trick
|
||||||
if configfile.has_key? "aptcachedir"
|
if configfile.has_key? "aptcachedir"
|
||||||
|
@ -23,10 +23,27 @@ if [ ! -x $ANDROID_HOME/tools/android ]; then
|
|||||||
unzip -qq $tools
|
unzip -qq $tools
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# disable the repositories of proprietary stuff
|
||||||
|
disabled="
|
||||||
|
@version@=1
|
||||||
|
@disabled@https\://dl.google.com/android/repository/extras/intel/addon.xml=disabled
|
||||||
|
@disabled@https\://dl.google.com/android/repository/glass/addon.xml=disabled
|
||||||
|
@disabled@https\://dl.google.com/android/repository/sys-img/android/sys-img.xml=disabled
|
||||||
|
@disabled@https\://dl.google.com/android/repository/sys-img/android-tv/sys-img.xml=disabled
|
||||||
|
@disabled@https\://dl.google.com/android/repository/sys-img/android-wear/sys-img.xml=disabled
|
||||||
|
@disabled@https\://dl.google.com/android/repository/sys-img/google_apis/sys-img.xml=disabled
|
||||||
|
"
|
||||||
|
test -d ${HOME}/.android || mkdir ${HOME}/.android
|
||||||
|
for line in $disabled; do
|
||||||
|
echo $line >> ${HOME}/.android/sites-settings.cfg
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
cd /vagrant/cache
|
cd /vagrant/cache
|
||||||
|
|
||||||
# make links for `android update sdk` to use and delete
|
# make links for `android update sdk` to use and delete
|
||||||
for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do
|
latestm2=`ls -1 android_m2repository*.zip | sort -n | tail -1`
|
||||||
|
for f in $latestm2 android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do
|
||||||
rm -f ${ANDROID_HOME}/temp/$f
|
rm -f ${ANDROID_HOME}/temp/$f
|
||||||
ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/
|
ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/
|
||||||
done
|
done
|
||||||
|
@ -11,6 +11,9 @@ sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.
|
|||||||
printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \
|
printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \
|
||||||
> /etc/apt/apt.conf.d/99no-install-recommends
|
> /etc/apt/apt.conf.d/99no-install-recommends
|
||||||
|
|
||||||
|
printf 'APT::Acquire::Retries "20";\n' \
|
||||||
|
> /etc/apt/apt.conf.d/99acquire-retries
|
||||||
|
|
||||||
if grep --quiet jessie /etc/apt/sources.list; then
|
if grep --quiet jessie /etc/apt/sources.list; then
|
||||||
echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list
|
echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list
|
||||||
fi
|
fi
|
||||||
|
@ -48,3 +48,11 @@
|
|||||||
# about the timeout, extend the timeout here. (default: 600 seconds)
|
# about the timeout, extend the timeout here. (default: 600 seconds)
|
||||||
#
|
#
|
||||||
# boot_timeout = 1200
|
# boot_timeout = 1200
|
||||||
|
|
||||||
|
# By default, this whole process uses VirtualBox as the provider, but
|
||||||
|
# QEMU+KVM is also supported via the libvirt plugin to vagrant. If
|
||||||
|
# this is run within a KVM guest, then libvirt's QEMU+KVM will be used
|
||||||
|
# automatically. It can also be manually enabled by uncommenting
|
||||||
|
# below:
|
||||||
|
#
|
||||||
|
# vm_provider = 'libvirt'
|
||||||
|
11
examples/public-read-only-s3-bucket-policy.json
Normal file
11
examples/public-read-only-s3-bucket-policy.json
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"Version":"2012-10-17",
|
||||||
|
"Statement":[
|
||||||
|
{"Sid":"AddPerm",
|
||||||
|
"Effect":"Allow",
|
||||||
|
"Principal":"*",
|
||||||
|
"Action":"s3:GetObject",
|
||||||
|
"Resource":"arn:aws:s3:::examplebucket/fdroid/*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
@ -1,3 +1,4 @@
|
|||||||
|
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
if [ `dirname $0` != "." ]; then
|
if [ `dirname $0` != "." ]; then
|
||||||
@ -11,14 +12,15 @@ fi
|
|||||||
|
|
||||||
# make sure that no VirtualBox processes are left running
|
# make sure that no VirtualBox processes are left running
|
||||||
cleanup_all() {
|
cleanup_all() {
|
||||||
|
set +e
|
||||||
echo "$(date -u) - cleanup in progress..."
|
echo "$(date -u) - cleanup in progress..."
|
||||||
ps auxww|grep VBox
|
ps auxww | grep -e VBox -e qemu
|
||||||
cd $WORKSPACE/buildserver
|
cd $WORKSPACE/buildserver
|
||||||
vagrant halt || true
|
vagrant halt
|
||||||
sleep 5
|
sleep 5
|
||||||
killall VBoxHeadless || true
|
killall VBoxHeadless
|
||||||
sleep 5
|
sleep 5
|
||||||
killall -9 VBoxHeadless || true
|
killall -9 VBoxHeadless
|
||||||
echo "$(date -u) - cleanup done."
|
echo "$(date -u) - cleanup done."
|
||||||
}
|
}
|
||||||
trap cleanup_all INT TERM EXIT
|
trap cleanup_all INT TERM EXIT
|
||||||
@ -37,8 +39,10 @@ sha256sum -c jessie64.box.sha256
|
|||||||
export XDG_CONFIG_HOME=$WORKSPACE
|
export XDG_CONFIG_HOME=$WORKSPACE
|
||||||
export VBOX_USER_HOME=$WORKSPACE/VirtualBox
|
export VBOX_USER_HOME=$WORKSPACE/VirtualBox
|
||||||
mkdir $VBOX_USER_HOME
|
mkdir $VBOX_USER_HOME
|
||||||
VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d
|
if which VBoxManage; then
|
||||||
VBoxManage setproperty logginglevel debug
|
VBoxManage setproperty machinefolder $WORKSPACE/virtualbox.d
|
||||||
|
VBoxManage setproperty logginglevel debug
|
||||||
|
fi
|
||||||
export VAGRANT_HOME=$WORKSPACE/vagrant.d
|
export VAGRANT_HOME=$WORKSPACE/vagrant.d
|
||||||
mkdir $VAGRANT_HOME
|
mkdir $VAGRANT_HOME
|
||||||
|
|
||||||
@ -46,7 +50,7 @@ cd $WORKSPACE
|
|||||||
echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
|
echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
|
||||||
echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
|
echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
|
||||||
echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
|
echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
|
||||||
./makebuildserver --verbose
|
./makebuildserver --verbose --clean
|
||||||
|
|
||||||
# this can be handled in the jenkins job, or here:
|
# this can be handled in the jenkins job, or here:
|
||||||
if [ -e fdroiddata ]; then
|
if [ -e fdroiddata ]; then
|
||||||
|
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
|
import stat
|
||||||
import sys
|
import sys
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
@ -63,8 +64,20 @@ config = {
|
|||||||
'cpus': 1,
|
'cpus': 1,
|
||||||
'memory': 1024,
|
'memory': 1024,
|
||||||
'hwvirtex': 'off',
|
'hwvirtex': 'off',
|
||||||
|
'vm_provider': 'virtualbox',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if os.path.isfile('/usr/bin/systemd-detect-virt'):
|
||||||
|
try:
|
||||||
|
virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
virt = 'none'
|
||||||
|
if virt == 'qemu' or virt == 'kvm':
|
||||||
|
print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
|
||||||
|
config['vm_provider'] = 'libvirt'
|
||||||
|
elif virt != 'none':
|
||||||
|
print('Running in an unsupported VM guest (' + virt + ')!')
|
||||||
|
|
||||||
# load config file, if present
|
# load config file, if present
|
||||||
if os.path.exists('makebuildserver.config.py'):
|
if os.path.exists('makebuildserver.config.py'):
|
||||||
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
|
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
|
||||||
@ -89,6 +102,15 @@ cachedir = config['cachedir']
|
|||||||
if not os.path.exists(cachedir):
|
if not os.path.exists(cachedir):
|
||||||
os.makedirs(cachedir, 0o755)
|
os.makedirs(cachedir, 0o755)
|
||||||
|
|
||||||
|
tmp = cachedir
|
||||||
|
while tmp != '/':
|
||||||
|
mode = os.stat(tmp).st_mode
|
||||||
|
if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
|
||||||
|
print('ERROR:', tmp, 'will not be accessible to the VM! To fix, run:')
|
||||||
|
print(' chmod a+X', tmp)
|
||||||
|
sys.exit(1)
|
||||||
|
tmp = os.path.dirname(tmp)
|
||||||
|
|
||||||
if config['apt_package_cache']:
|
if config['apt_package_cache']:
|
||||||
config['aptcachedir'] = cachedir + '/apt/archives'
|
config['aptcachedir'] = cachedir + '/apt/archives'
|
||||||
|
|
||||||
@ -338,6 +360,7 @@ elif os.path.exists('/proc/cpuinfo'):
|
|||||||
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
||||||
writevf = True
|
writevf = True
|
||||||
if os.path.exists(vf):
|
if os.path.exists(vf):
|
||||||
|
print('Halting', serverdir)
|
||||||
vagrant(['halt'], serverdir)
|
vagrant(['halt'], serverdir)
|
||||||
with open(vf, 'r', encoding='utf-8') as f:
|
with open(vf, 'r', encoding='utf-8') as f:
|
||||||
oldconfig = yaml.load(f)
|
oldconfig = yaml.load(f)
|
||||||
@ -353,6 +376,33 @@ if writevf:
|
|||||||
with open(vf, 'w', encoding='utf-8') as f:
|
with open(vf, 'w', encoding='utf-8') as f:
|
||||||
yaml.dump(config, f)
|
yaml.dump(config, f)
|
||||||
|
|
||||||
|
if config['vm_provider'] == 'libvirt':
|
||||||
|
returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose)
|
||||||
|
found_basebox = False
|
||||||
|
needs_mutate = False
|
||||||
|
for line in out.splitlines():
|
||||||
|
if line.startswith(config['basebox']):
|
||||||
|
found_basebox = True
|
||||||
|
if line.split('(')[1].split(',')[0] != 'libvirt':
|
||||||
|
needs_mutate = True
|
||||||
|
continue
|
||||||
|
if not found_basebox:
|
||||||
|
if isinstance(config['baseboxurl'], str):
|
||||||
|
baseboxurl = config['baseboxurl']
|
||||||
|
else:
|
||||||
|
baseboxurl = config['baseboxurl'][0]
|
||||||
|
print('Adding', config['basebox'], 'from', baseboxurl)
|
||||||
|
vagrant(['box', 'add', '--name', config['basebox'], baseboxurl],
|
||||||
|
serverdir, printout=options.verbose)
|
||||||
|
needs_mutate = True
|
||||||
|
if needs_mutate:
|
||||||
|
print('Converting', config['basebox'], 'to libvirt format')
|
||||||
|
vagrant(['mutate', config['basebox'], 'libvirt'],
|
||||||
|
serverdir, printout=options.verbose)
|
||||||
|
print('Removing virtualbox format copy of', config['basebox'])
|
||||||
|
vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']],
|
||||||
|
serverdir, printout=options.verbose)
|
||||||
|
|
||||||
print("Configuring build server VM")
|
print("Configuring build server VM")
|
||||||
returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)
|
returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)
|
||||||
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
||||||
|
@ -5,6 +5,7 @@
|
|||||||
import inspect
|
import inspect
|
||||||
import optparse
|
import optparse
|
||||||
import os
|
import os
|
||||||
|
import requests
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
@ -30,6 +31,12 @@ class ImportTest(unittest.TestCase):
|
|||||||
fdroidserver.common.config = config
|
fdroidserver.common.config = config
|
||||||
|
|
||||||
url = 'https://gitlab.com/fdroid/fdroidclient'
|
url = 'https://gitlab.com/fdroid/fdroidclient'
|
||||||
|
r = requests.head(url)
|
||||||
|
if r.status_code != 200:
|
||||||
|
print("ERROR", url, 'unreachable (', r.status_code, ')')
|
||||||
|
print('Skipping ImportTest!')
|
||||||
|
return
|
||||||
|
|
||||||
app = fdroidserver.metadata.get_default_app_info()
|
app = fdroidserver.metadata.get_default_app_info()
|
||||||
app.UpdateCheckMode = "Tags"
|
app.UpdateCheckMode = "Tags"
|
||||||
root_dir, src_dir = import_proxy.get_metadata_from_url(app, url)
|
root_dir, src_dir = import_proxy.get_metadata_from_url(app, url)
|
||||||
|
Loading…
Reference in New Issue
Block a user