kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-10-02 09:10:11 +02:00
Code Activity

Merge branch 'move-makebuildserver.config.py-to-Vagrantfile.yaml' into 'master'

Browse Source 
buildserver: turn ./makebuildserver into a pure helper script, allow standalone vagrant operation

See merge request fdroid/fdroidserver!1222
This commit is contained in:
Michael Pöhn 2022-11-03 13:01:18 +00:00
commit f8395ffe71
8 changed files with 123 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitlab-ci.yml
View File

@ -223,7 +223,6 @@ black:
- pip install black
- black --check --diff --color
examples/fdroid_extract_repo_pubkey.py
examples/makebuildserver.config.py
fdroid
fdroidserver/btlog.py
fdroidserver/exception.py

3
MANIFEST.in
View File

@ -14,10 +14,10 @@ include examples/fdroid_export_keystore_to_nitrokey.py
include examples/fdroid_extract_repo_pubkey.py
include examples/fdroid_fetchsrclibs.py
include examples/fdroid_nitrokeyimport.py
include examples/makebuildserver.config.py
include examples/opensc-fdroid.cfg
include examples/public-read-only-s3-bucket-policy.json
include examples/template.yml
include examples/Vagrantfile.yaml
include gradlew-fdroid
include LICENSE
include locale/bo/LC_MESSAGES/fdroidserver.po
@ -42,7 +42,6 @@ include locale/zh_Hans/LC_MESSAGES/fdroidserver.po
include locale/zh_Hant/LC_MESSAGES/fdroidserver.po
include makebuildserver
include README.md
include tests/androguard_test.py
include tests/apk.embedded_1.apk
include tests/bad-unicode-*.apk
include tests/build.TestCase

45
buildserver/Vagrantfile vendored
View File

@ -1,29 +1,41 @@
require 'yaml'
require 'pathname'
require 'fileutils'
configfile = {
'boot_timeout' => 600,
'cachedir' => File.join(ENV['HOME'], '.cache', 'fdroidserver'),
'cpus' => 1,
'debian_mirror' => 'https://deb.debian.org/debian/',
'hwvirtex' => 'on',
'memory' => 2048,
'vm_provider' => 'virtualbox',
}
srvpath = Pathname.new(File.dirname(__FILE__)).realpath
configpath = File.join(srvpath, "/Vagrantfile.yaml")
if File.exists? configpath
configfile = YAML.load_file(configpath)
c = YAML.load_file(configpath)
if c and not c.empty?
c.each do |k,v|
configfile[k] = v
end
end
else
puts "#{configpath} does not exist, run ./makebuildserver?"
configfile = Hash.new
puts "Copying example file to #{configpath}"
FileUtils.cp('../examples/Vagrantfile.yaml', configpath)
end
Vagrant.configure("2") do |config|
# these two caching methods conflict, so only use one at a time
if Vagrant.has_plugin?("vagrant-cachier") and not configfile.has_key? "aptcachedir"
if Vagrant.has_plugin?("vagrant-cachier")
config.cache.scope = :box
config.cache.auto_detect = false
config.cache.enable :apt
config.cache.enable :chef
end
config.vm.box = configfile['basebox']
if configfile.has_key? "basebox_version"
config.vm.box_version = configfile['basebox_version']
end
config.vm.box = "fdroid/bullseye64"
if not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox"
# default to VirtualBox if not set
@ -66,23 +78,14 @@ Vagrant.configure("2") do |config|
args: [configfile["aptproxy"]]
end
# buildserver/ is shared to the VM's /vagrant by default so the old
# default does not need a custom mount
if configfile["cachedir"] != "buildserver/cache"
config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
create: true, type: synced_folder_type
end
config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
create: true, type: synced_folder_type
# Make sure dir exists to mount to, since buildserver/ is
# automatically mounted as /vagrant in the guest VM. This is more
# necessary with 9p synced folders
Dir.mkdir('cache') unless File.exists?('cache')
# cache .deb packages on the host via a mount trick
if configfile.has_key? "aptcachedir"
config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives",
owner: 'root', group: 'root', create: true
end
config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars",
args: ["/opt/android-sdk"]
config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install",

2
buildserver/provision-apt-get-install
View File

@ -90,6 +90,8 @@ purge="
xxd
xz-utils
"
# clean up files packages to be purged, then purge the packages
rm -rf /var/run/dbus /var/log/unattended-upgrades
apt-get purge $purge
apt-get upgrade --download-only

53
examples/Vagrantfile.yaml Normal file
View File

@ -0,0 +1,53 @@
# You may want to alter these before running ./makebuildserver
# In the process of setting up the build server, many gigs of files
# are downloaded (Android SDK components, gradle, etc). These are
# cached so that they are not redownloaded each time. By default,
# these are stored in ~/.cache/fdroidserver
#
# cachedir: buildserver/cache
# To specify which Debian mirror the build server VM should use, by
# default it uses http.debian.net, which auto-detects which is the
# best mirror to use.
#
# debian_mirror: https://debian.osuosl.org/debian/
# The amount of RAM the build server will have (default: 2048)
# memory: 3584
# The number of CPUs the build server will have
# cpus: 1
# Debian package proxy server - if you have one
# aptproxy: http://192.168.0.19:8000
# If this is running on an older machine or on a virtualized system,
# it can run a lot slower. If the provisioning fails with a warning
# about the timeout, extend the timeout here. (default: 600 seconds)
#
# boot_timeout: 1200
# By default, this whole process uses VirtualBox as the provider, but
# QEMU+KVM is also supported via the libvirt plugin to vagrant. If
# this is run within a KVM guest, then libvirt's QEMU+KVM will be used
# automatically. It can also be manually enabled by uncommenting
# below:
#
# vm_provider: libvirt
# By default libvirt uses 'virtio' for both network and disk drivers.
# Some systems (eg. nesting VMware ESXi) do not support virtio. As a
# workaround for such rare cases, this setting allows to configure
# KVM/libvirt to emulate hardware rather than using virtio.
#
# libvirt_disk_bus: sata
# libvirt_nic_model_type: rtl8139
# Sometimes, it is not possible to use the 9p synced folder type with
# libvirt, like if running a KVM buildserver instance inside of a
# VMware ESXi guest. In that case, using NFS or another method is
# required.
#
# synced_folder_type: nfs

94
examples/makebuildserver.config.py
View File

@ -1,94 +0,0 @@
#!/usr/bin/env python3
#
# You may want to alter these before running ./makebuildserver
# Name of the Vagrant basebox to use, by default it will be downloaded
# from Vagrant Cloud. For release builds setup, generate the basebox
# locally using https://gitlab.com/fdroid/basebox, add it to Vagrant,
# then set this to the local basebox name.
# This defaults to "fdroid/basebox-stretch64" which will download a
# prebuilt basebox from https://app.vagrantup.com/fdroid.
#
# (If you change this value you have to supply the `--clean` option on
# your next `makebuildserver` run.)
#
# basebox = "basebox-stretch64"
# This allows you to pin your basebox to a specific versions. It defaults
# the most recent basebox version which can be aumotaically verifyed by
# `makebuildserver`.
# Please note that vagrant does not support versioning of locally added
# boxes, so we can't support that either.
#
# (If you change this value you have to supply the `--clean` option on
# your next `makebuildserver` run.)
#
# basebox_version = "0.1"
# In the process of setting up the build server, many gigs of files
# are downloaded (Android SDK components, gradle, etc). These are
# cached so that they are not redownloaded each time. By default,
# these are stored in ~/.cache/fdroidserver
#
# cachedir = 'buildserver/cache'
# A big part of creating a new instance is downloading packages from Debian.
# This setups up a folder in ~/.cache/fdroidserver to cache the downloaded
# packages when rebuilding the build server from scratch. This requires
# that virtualbox-guest-utils is installed.
#
# apt_package_cache = True
# The buildserver can use some local caches to speed up builds,
# especially when the internet connection is slow and/or expensive.
# If enabled, the buildserver setup will look for standard caches in
# your HOME dir and copy them to the buildserver VM. Be aware: this
# will reduce the isolation of the buildserver from your host machine,
# so the buildserver will provide an environment only as trustworthy
# as the host machine's environment.
#
# copy_caches_from_host = True
# To specify which Debian mirror the build server VM should use, by
# default it uses http.debian.net, which auto-detects which is the
# best mirror to use.
#
# debian_mirror = 'http://ftp.uk.debian.org/debian/'
# The amount of RAM the build server will have (default: 2048)
# memory = 3584
# The number of CPUs the build server will have
# cpus = 1
# Debian package proxy server - if you have one
# aptproxy = "http://192.168.0.19:8000"
# If this is running on an older machine or on a virtualized system,
# it can run a lot slower. If the provisioning fails with a warning
# about the timeout, extend the timeout here. (default: 600 seconds)
#
# boot_timeout = 1200
# By default, this whole process uses VirtualBox as the provider, but
# QEMU+KVM is also supported via the libvirt plugin to vagrant. If
# this is run within a KVM guest, then libvirt's QEMU+KVM will be used
# automatically. It can also be manually enabled by uncommenting
# below:
#
# vm_provider = 'libvirt'
# By default libvirt uses 'virtio' for both network and disk drivers.
# Some systems (eg. nesting VMware ESXi) do not support virtio. As a
# workaround for such rare cases, this setting allows to configure
# KVM/libvirt to emulate hardware rather than using virtio.
#
# libvirt_disk_bus = 'sata'
# libvirt_nic_model_type = 'rtl8139'
# Sometimes, it is not possible to use the 9p synced folder type with
# libvirt, like if running a KVM buildserver instance inside of a
# VMware ESXi guest. In that case, using NFS or another method is
# required.
#
# synced_folder_type = 'nfs'

12
jenkins-setup-build-environment
View File

@ -66,13 +66,11 @@ if [ `nproc` -le 6 ]; then
else
cpus=6
fi
cat <<EOF > $WORKSPACE/makebuildserver.config.py
debian_mirror = 'http://deb.debian.org/debian/'
boot_timeout = 1200
apt_package_cache = True
copy_caches_from_host = True
memory = $memory
cpus = $cpus
cat <<EOF > $WORKSPACE/buildserver/Vagrantfile.yaml
debian_mirror: https://deb.debian.org/debian/
boot_timeout: 1200
memory: $memory
cpus: $cpus
EOF
cd $WORKSPACE

102
makebuildserver
View File

@ -24,9 +24,13 @@ parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1
parser.add_option('-q', action='store_const', const=0, dest='verbosity')
parser.add_option("-c", "--clean", action="store_true", default=False,
help="Build from scratch, rather than attempting to update the existing server")
parser.add_option('--skip-box-verification', action="store_true", default=False,
help="""Skip verifying the downloaded base box.""")
parser.add_option('--skip-cache-update', action="store_true", default=False,
help="""Skip downloading and checking cache."""
"""This assumes that the cache is already downloaded completely.""")
parser.add_option('--copy-caches-from-host', action="store_true", default=False,
help="""Copy gradle and pip caches into the buildserver VM""")
parser.add_option('--keep-box-file', action="store_true", default=False,
help="""Box file will not be deleted after adding it to box storage"""
""" (KVM-only).""")
@ -74,36 +78,35 @@ BASEBOX_CHECKSUMS = {
},
}
config = {
'basebox': BASEBOX_DEFAULT,
'debian_mirror': 'https://deb.debian.org/debian/',
'apt_package_cache': False,
'copy_caches_from_host': False,
'boot_timeout': 600,
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
'cpus': 1,
'memory': 2048,
'hwvirtex': 'off',
'vm_provider': 'virtualbox',
}
configfile = 'buildserver/Vagrantfile.yaml'
if not os.path.exists(configfile):
logging.warning('%s does not exist, copying template file.' % configfile)
shutil.copy('examples/Vagrantfile.yaml', configfile)
with open(configfile) as fp:
config = yaml.safe_load(fp)
with open('buildserver/Vagrantfile') as fp:
m = re.search(r"""\.vm\.box\s*=\s*["'](.*)["']""", fp.read())
if not m:
logging.error('Cannot find box name in buildserver/Vagrantfile!')
exit(1)
config['basebox'] = m.group(1)
config['basebox_version'] = BASEBOX_VERSION_DEFAULT
config['cachedir'] = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
# load config file, if present
show_config_deprecation = False
if os.path.exists('makebuildserver.config.py'):
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
show_config_deprecation = True
logging.error('makebuildserver.config.py exists!')
elif os.path.exists('makebs.config.py'):
show_config_deprecation = True
# this is the old name for the config file
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
if '__builtins__' in config:
del config['__builtins__'] # added by compile/exec
logging.error('makebs.config.py exists!')
if show_config_deprecation:
logging.error('Config is via buildserver/Vagrantfile.yaml and command line flags.')
parser.print_help()
exit(1)
logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config:
config['basebox_version'] = BASEBOX_VERSION_DEFAULT
# note: vagrant allows putting '/' into the name of a local box,
# so this check is not completely reliable, but better than nothing
if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']:
logging.critical("Can not get version '{version}' for basebox '{box}', "
"vagrant does not support versioning for locally added boxes."
.format(box=config['basebox'], version=config['basebox_version']))
# Update cached files.
if not os.path.exists(config['cachedir']):
@ -121,18 +124,6 @@ if config['vm_provider'] == 'libvirt':
tmp = os.path.dirname(tmp)
logging.debug('cache dir %s is accessible for libvirt vm.', config['cachedir'])
if config['apt_package_cache']:
config['aptcachedir'] = config['cachedir'] + '/apt/archives'
logging.debug('aptcachedir is set to %s', config['aptcachedir'])
aptcachelock = os.path.join(config['aptcachedir'], 'lock')
if os.path.isfile(aptcachelock):
logging.info('apt cache dir is locked, removing lock')
os.remove(aptcachelock)
aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
if os.path.isdir(aptcachepartial):
logging.info('removing partial downloads from apt cache dir')
shutil.rmtree(aptcachepartial)
CACHE_FILES = [
('https://services.gradle.org/distributions/gradle-6.8.3-bin.zip',
'7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205'),
@ -265,17 +256,14 @@ def main():
# use VirtualBox software virtualization if hardware is not available,
# like if this is being run in kvm or some other VM platform, like
# http://jenkins.debian.net, the values are 'on' or 'off'
if sys.platform.startswith('darwin'):
# all < 10 year old Macs work, and OSX servers as VM host are very
# rare, but this could also be auto-detected if someone codes it
config['hwvirtex'] = 'on'
logging.info('platform is darwnin -> hwvirtex = \'on\'')
elif os.path.exists('/proc/cpuinfo'):
if config.get('hwvirtex') != 'off' and os.path.exists('/proc/cpuinfo'):
with open('/proc/cpuinfo') as f:
contents = f.read()
if 'vmx' in contents or 'svm' in contents:
config['hwvirtex'] = 'on'
logging.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
logging.debug('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
else:
logging.error('hwvirtex = \'on\' and no \'vmx\' or \'svm\' found in /proc/cpuinfo!')
exit(1)
serverdir = os.path.join(os.getcwd(), 'buildserver')
logfilename = os.path.join(serverdir, 'up.log')
@ -324,27 +312,15 @@ def main():
"virtualbox, libvirt)"
.format(vm_provider=config['cm_provider']))
sys.exit(1)
# Check if selected Vagrant box is available
available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
logging.critical("Vagrant box '{basebox}' not available "
"for '{vm_provider}' VM provider. "
"Please make sure it's added to vagrant. "
"(If you need a basebox to begin with, "
"here is how we're bootstrapping it: "
"https://gitlab.com/fdroid/basebox)"
.format(vm_provider=config['vm_provider'],
basebox=config['basebox']))
sys.exit(1)
# Download and verify pre-built Vagrant boxes
if config['basebox'] == BASEBOX_DEFAULT:
if not options.skip_box_verification:
buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default'])
if buildserver_not_created or options.clean:
# make vagrant download and add basebox
target_basebox_installed = any([x for x in v.box_list() if x.name == BASEBOX_DEFAULT and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
target_basebox_installed = any([x for x in v.box_list() if x.name == config['basebox'] and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
if not target_basebox_installed:
cmd = [shutil.which('vagrant'), 'box', 'add', BASEBOX_DEFAULT,
cmd = [shutil.which('vagrant'), 'box', 'add', config['basebox'],
'--box-version=' + config['basebox_version'],
'--provider=' + config['vm_provider']]
ret_val = subprocess.call(cmd)
@ -365,7 +341,7 @@ def main():
for filename, sha256 in BASEBOX_CHECKSUMS[config['basebox_version']][config['vm_provider']].items():
verify_file_sha256(os.path.join(get_vagrant_home(),
'boxes',
BASEBOX_DEFAULT.replace('/', '-VAGRANTSLASH-'),
config['basebox'].replace('/', '-VAGRANTSLASH-'),
config['basebox_version'],
config['vm_provider'],
filename),
@ -375,8 +351,6 @@ def main():
.format(box=config['basebox'],
provider=config['vm_provider'],
version=config['basebox_version']))
else:
logging.debug('not updating basebox ...')
else:
logging.debug('using unverified basebox ...')
@ -389,7 +363,7 @@ def main():
logging.error("'vagrant up' failed.")
sys.exit(1)
if config['copy_caches_from_host']:
if options.copy_caches_from_host:
ssh_config = v.ssh_config()
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)