diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fe44dba..3a4f230b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -223,7 +223,6 @@ black: - pip install black - black --check --diff --color examples/fdroid_extract_repo_pubkey.py - examples/makebuildserver.config.py fdroid fdroidserver/btlog.py fdroidserver/exception.py diff --git a/MANIFEST.in b/MANIFEST.in index 297dae0b..3adec161 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,10 +14,10 @@ include examples/fdroid_export_keystore_to_nitrokey.py include examples/fdroid_extract_repo_pubkey.py include examples/fdroid_fetchsrclibs.py include examples/fdroid_nitrokeyimport.py -include examples/makebuildserver.config.py include examples/opensc-fdroid.cfg include examples/public-read-only-s3-bucket-policy.json include examples/template.yml +include examples/Vagrantfile.yaml include gradlew-fdroid include LICENSE include locale/bo/LC_MESSAGES/fdroidserver.po @@ -42,7 +42,6 @@ include locale/zh_Hans/LC_MESSAGES/fdroidserver.po include locale/zh_Hant/LC_MESSAGES/fdroidserver.po include makebuildserver include README.md -include tests/androguard_test.py include tests/apk.embedded_1.apk include tests/bad-unicode-*.apk include tests/build.TestCase diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 8caa3296..f5172870 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -1,29 +1,41 @@ require 'yaml' require 'pathname' +require 'fileutils' + +configfile = { + 'boot_timeout' => 600, + 'cachedir' => File.join(ENV['HOME'], '.cache', 'fdroidserver'), + 'cpus' => 1, + 'debian_mirror' => 'https://deb.debian.org/debian/', + 'hwvirtex' => 'on', + 'memory' => 2048, + 'vm_provider' => 'virtualbox', +} srvpath = Pathname.new(File.dirname(__FILE__)).realpath configpath = File.join(srvpath, "/Vagrantfile.yaml") if File.exists? configpath - configfile = YAML.load_file(configpath) + c = YAML.load_file(configpath) + if c and not c.empty? + c.each do |k,v| + configfile[k] = v + end + end else - puts "#{configpath} does not exist, run ./makebuildserver?" - configfile = Hash.new + puts "Copying example file to #{configpath}" + FileUtils.cp('../examples/Vagrantfile.yaml', configpath) end Vagrant.configure("2") do |config| - # these two caching methods conflict, so only use one at a time - if Vagrant.has_plugin?("vagrant-cachier") and not configfile.has_key? "aptcachedir" + if Vagrant.has_plugin?("vagrant-cachier") config.cache.scope = :box config.cache.auto_detect = false config.cache.enable :apt config.cache.enable :chef end - config.vm.box = configfile['basebox'] - if configfile.has_key? "basebox_version" - config.vm.box_version = configfile['basebox_version'] - end + config.vm.box = "fdroid/bullseye64" if not configfile.has_key? "vm_provider" or configfile["vm_provider"] == "virtualbox" # default to VirtualBox if not set @@ -66,23 +78,14 @@ Vagrant.configure("2") do |config| args: [configfile["aptproxy"]] end - # buildserver/ is shared to the VM's /vagrant by default so the old - # default does not need a custom mount - if configfile["cachedir"] != "buildserver/cache" - config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', - create: true, type: synced_folder_type - end + config.vm.synced_folder configfile["cachedir"], '/vagrant/cache', + create: true, type: synced_folder_type + # Make sure dir exists to mount to, since buildserver/ is # automatically mounted as /vagrant in the guest VM. This is more # necessary with 9p synced folders Dir.mkdir('cache') unless File.exists?('cache') - # cache .deb packages on the host via a mount trick - if configfile.has_key? "aptcachedir" - config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives", - owner: 'root', group: 'root', create: true - end - config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars", args: ["/opt/android-sdk"] config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install", diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index b369d36f..b856562c 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -90,6 +90,8 @@ purge=" xxd xz-utils " +# clean up files packages to be purged, then purge the packages +rm -rf /var/run/dbus /var/log/unattended-upgrades apt-get purge $purge apt-get upgrade --download-only diff --git a/examples/Vagrantfile.yaml b/examples/Vagrantfile.yaml new file mode 100644 index 00000000..6bf9c5ca --- /dev/null +++ b/examples/Vagrantfile.yaml @@ -0,0 +1,53 @@ + +# You may want to alter these before running ./makebuildserver + +# In the process of setting up the build server, many gigs of files +# are downloaded (Android SDK components, gradle, etc). These are +# cached so that they are not redownloaded each time. By default, +# these are stored in ~/.cache/fdroidserver +# +# cachedir: buildserver/cache + +# To specify which Debian mirror the build server VM should use, by +# default it uses http.debian.net, which auto-detects which is the +# best mirror to use. +# +# debian_mirror: https://debian.osuosl.org/debian/ + +# The amount of RAM the build server will have (default: 2048) +# memory: 3584 + +# The number of CPUs the build server will have +# cpus: 1 + +# Debian package proxy server - if you have one +# aptproxy: http://192.168.0.19:8000 + +# If this is running on an older machine or on a virtualized system, +# it can run a lot slower. If the provisioning fails with a warning +# about the timeout, extend the timeout here. (default: 600 seconds) +# +# boot_timeout: 1200 + +# By default, this whole process uses VirtualBox as the provider, but +# QEMU+KVM is also supported via the libvirt plugin to vagrant. If +# this is run within a KVM guest, then libvirt's QEMU+KVM will be used +# automatically. It can also be manually enabled by uncommenting +# below: +# +# vm_provider: libvirt + +# By default libvirt uses 'virtio' for both network and disk drivers. +# Some systems (eg. nesting VMware ESXi) do not support virtio. As a +# workaround for such rare cases, this setting allows to configure +# KVM/libvirt to emulate hardware rather than using virtio. +# +# libvirt_disk_bus: sata +# libvirt_nic_model_type: rtl8139 + +# Sometimes, it is not possible to use the 9p synced folder type with +# libvirt, like if running a KVM buildserver instance inside of a +# VMware ESXi guest. In that case, using NFS or another method is +# required. +# +# synced_folder_type: nfs diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py deleted file mode 100644 index cb47f95f..00000000 --- a/examples/makebuildserver.config.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# -# You may want to alter these before running ./makebuildserver - -# Name of the Vagrant basebox to use, by default it will be downloaded -# from Vagrant Cloud. For release builds setup, generate the basebox -# locally using https://gitlab.com/fdroid/basebox, add it to Vagrant, -# then set this to the local basebox name. -# This defaults to "fdroid/basebox-stretch64" which will download a -# prebuilt basebox from https://app.vagrantup.com/fdroid. -# -# (If you change this value you have to supply the `--clean` option on -# your next `makebuildserver` run.) -# -# basebox = "basebox-stretch64" - -# This allows you to pin your basebox to a specific versions. It defaults -# the most recent basebox version which can be aumotaically verifyed by -# `makebuildserver`. -# Please note that vagrant does not support versioning of locally added -# boxes, so we can't support that either. -# -# (If you change this value you have to supply the `--clean` option on -# your next `makebuildserver` run.) -# -# basebox_version = "0.1" - -# In the process of setting up the build server, many gigs of files -# are downloaded (Android SDK components, gradle, etc). These are -# cached so that they are not redownloaded each time. By default, -# these are stored in ~/.cache/fdroidserver -# -# cachedir = 'buildserver/cache' - -# A big part of creating a new instance is downloading packages from Debian. -# This setups up a folder in ~/.cache/fdroidserver to cache the downloaded -# packages when rebuilding the build server from scratch. This requires -# that virtualbox-guest-utils is installed. -# -# apt_package_cache = True - -# The buildserver can use some local caches to speed up builds, -# especially when the internet connection is slow and/or expensive. -# If enabled, the buildserver setup will look for standard caches in -# your HOME dir and copy them to the buildserver VM. Be aware: this -# will reduce the isolation of the buildserver from your host machine, -# so the buildserver will provide an environment only as trustworthy -# as the host machine's environment. -# -# copy_caches_from_host = True - -# To specify which Debian mirror the build server VM should use, by -# default it uses http.debian.net, which auto-detects which is the -# best mirror to use. -# -# debian_mirror = 'http://ftp.uk.debian.org/debian/' - -# The amount of RAM the build server will have (default: 2048) -# memory = 3584 - -# The number of CPUs the build server will have -# cpus = 1 - -# Debian package proxy server - if you have one -# aptproxy = "http://192.168.0.19:8000" - -# If this is running on an older machine or on a virtualized system, -# it can run a lot slower. If the provisioning fails with a warning -# about the timeout, extend the timeout here. (default: 600 seconds) -# -# boot_timeout = 1200 - -# By default, this whole process uses VirtualBox as the provider, but -# QEMU+KVM is also supported via the libvirt plugin to vagrant. If -# this is run within a KVM guest, then libvirt's QEMU+KVM will be used -# automatically. It can also be manually enabled by uncommenting -# below: -# -# vm_provider = 'libvirt' - -# By default libvirt uses 'virtio' for both network and disk drivers. -# Some systems (eg. nesting VMware ESXi) do not support virtio. As a -# workaround for such rare cases, this setting allows to configure -# KVM/libvirt to emulate hardware rather than using virtio. -# -# libvirt_disk_bus = 'sata' -# libvirt_nic_model_type = 'rtl8139' - -# Sometimes, it is not possible to use the 9p synced folder type with -# libvirt, like if running a KVM buildserver instance inside of a -# VMware ESXi guest. In that case, using NFS or another method is -# required. -# -# synced_folder_type = 'nfs' diff --git a/jenkins-setup-build-environment b/jenkins-setup-build-environment index 489717b2..95c2049a 100755 --- a/jenkins-setup-build-environment +++ b/jenkins-setup-build-environment @@ -66,13 +66,11 @@ if [ `nproc` -le 6 ]; then else cpus=6 fi -cat < $WORKSPACE/makebuildserver.config.py -debian_mirror = 'http://deb.debian.org/debian/' -boot_timeout = 1200 -apt_package_cache = True -copy_caches_from_host = True -memory = $memory -cpus = $cpus +cat < $WORKSPACE/buildserver/Vagrantfile.yaml +debian_mirror: https://deb.debian.org/debian/ +boot_timeout: 1200 +memory: $memory +cpus: $cpus EOF cd $WORKSPACE diff --git a/makebuildserver b/makebuildserver index 21432a38..4af88601 100755 --- a/makebuildserver +++ b/makebuildserver @@ -24,9 +24,13 @@ parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1 parser.add_option('-q', action='store_const', const=0, dest='verbosity') parser.add_option("-c", "--clean", action="store_true", default=False, help="Build from scratch, rather than attempting to update the existing server") +parser.add_option('--skip-box-verification', action="store_true", default=False, + help="""Skip verifying the downloaded base box.""") parser.add_option('--skip-cache-update', action="store_true", default=False, help="""Skip downloading and checking cache.""" """This assumes that the cache is already downloaded completely.""") +parser.add_option('--copy-caches-from-host', action="store_true", default=False, + help="""Copy gradle and pip caches into the buildserver VM""") parser.add_option('--keep-box-file', action="store_true", default=False, help="""Box file will not be deleted after adding it to box storage""" """ (KVM-only).""") @@ -74,36 +78,35 @@ BASEBOX_CHECKSUMS = { }, } -config = { - 'basebox': BASEBOX_DEFAULT, - 'debian_mirror': 'https://deb.debian.org/debian/', - 'apt_package_cache': False, - 'copy_caches_from_host': False, - 'boot_timeout': 600, - 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), - 'cpus': 1, - 'memory': 2048, - 'hwvirtex': 'off', - 'vm_provider': 'virtualbox', -} +configfile = 'buildserver/Vagrantfile.yaml' +if not os.path.exists(configfile): + logging.warning('%s does not exist, copying template file.' % configfile) + shutil.copy('examples/Vagrantfile.yaml', configfile) +with open(configfile) as fp: + config = yaml.safe_load(fp) +with open('buildserver/Vagrantfile') as fp: + m = re.search(r"""\.vm\.box\s*=\s*["'](.*)["']""", fp.read()) + if not m: + logging.error('Cannot find box name in buildserver/Vagrantfile!') + exit(1) + config['basebox'] = m.group(1) +config['basebox_version'] = BASEBOX_VERSION_DEFAULT +config['cachedir'] = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver') -# load config file, if present +show_config_deprecation = False if os.path.exists('makebuildserver.config.py'): - exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config) + show_config_deprecation = True + logging.error('makebuildserver.config.py exists!') elif os.path.exists('makebs.config.py'): + show_config_deprecation = True # this is the old name for the config file - exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config) -if '__builtins__' in config: - del config['__builtins__'] # added by compile/exec + logging.error('makebs.config.py exists!') +if show_config_deprecation: + logging.error('Config is via buildserver/Vagrantfile.yaml and command line flags.') + parser.print_help() + exit(1) + logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True)) -if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config: - config['basebox_version'] = BASEBOX_VERSION_DEFAULT -# note: vagrant allows putting '/' into the name of a local box, -# so this check is not completely reliable, but better than nothing -if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']: - logging.critical("Can not get version '{version}' for basebox '{box}', " - "vagrant does not support versioning for locally added boxes." - .format(box=config['basebox'], version=config['basebox_version'])) # Update cached files. if not os.path.exists(config['cachedir']): @@ -121,18 +124,6 @@ if config['vm_provider'] == 'libvirt': tmp = os.path.dirname(tmp) logging.debug('cache dir %s is accessible for libvirt vm.', config['cachedir']) -if config['apt_package_cache']: - config['aptcachedir'] = config['cachedir'] + '/apt/archives' - logging.debug('aptcachedir is set to %s', config['aptcachedir']) - aptcachelock = os.path.join(config['aptcachedir'], 'lock') - if os.path.isfile(aptcachelock): - logging.info('apt cache dir is locked, removing lock') - os.remove(aptcachelock) - aptcachepartial = os.path.join(config['aptcachedir'], 'partial') - if os.path.isdir(aptcachepartial): - logging.info('removing partial downloads from apt cache dir') - shutil.rmtree(aptcachepartial) - CACHE_FILES = [ ('https://services.gradle.org/distributions/gradle-6.8.3-bin.zip', '7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205'), @@ -265,17 +256,14 @@ def main(): # use VirtualBox software virtualization if hardware is not available, # like if this is being run in kvm or some other VM platform, like # http://jenkins.debian.net, the values are 'on' or 'off' - if sys.platform.startswith('darwin'): - # all < 10 year old Macs work, and OSX servers as VM host are very - # rare, but this could also be auto-detected if someone codes it - config['hwvirtex'] = 'on' - logging.info('platform is darwnin -> hwvirtex = \'on\'') - elif os.path.exists('/proc/cpuinfo'): + if config.get('hwvirtex') != 'off' and os.path.exists('/proc/cpuinfo'): with open('/proc/cpuinfo') as f: contents = f.read() if 'vmx' in contents or 'svm' in contents: - config['hwvirtex'] = 'on' - logging.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'') + logging.debug('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'') + else: + logging.error('hwvirtex = \'on\' and no \'vmx\' or \'svm\' found in /proc/cpuinfo!') + exit(1) serverdir = os.path.join(os.getcwd(), 'buildserver') logfilename = os.path.join(serverdir, 'up.log') @@ -324,27 +312,15 @@ def main(): "virtualbox, libvirt)" .format(vm_provider=config['cm_provider'])) sys.exit(1) - # Check if selected Vagrant box is available - available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']] - if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider: - logging.critical("Vagrant box '{basebox}' not available " - "for '{vm_provider}' VM provider. " - "Please make sure it's added to vagrant. " - "(If you need a basebox to begin with, " - "here is how we're bootstrapping it: " - "https://gitlab.com/fdroid/basebox)" - .format(vm_provider=config['vm_provider'], - basebox=config['basebox'])) - sys.exit(1) # Download and verify pre-built Vagrant boxes - if config['basebox'] == BASEBOX_DEFAULT: + if not options.skip_box_verification: buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default']) if buildserver_not_created or options.clean: # make vagrant download and add basebox - target_basebox_installed = any([x for x in v.box_list() if x.name == BASEBOX_DEFAULT and x.provider == config['vm_provider'] and x.version == config['basebox_version']]) + target_basebox_installed = any([x for x in v.box_list() if x.name == config['basebox'] and x.provider == config['vm_provider'] and x.version == config['basebox_version']]) if not target_basebox_installed: - cmd = [shutil.which('vagrant'), 'box', 'add', BASEBOX_DEFAULT, + cmd = [shutil.which('vagrant'), 'box', 'add', config['basebox'], '--box-version=' + config['basebox_version'], '--provider=' + config['vm_provider']] ret_val = subprocess.call(cmd) @@ -365,7 +341,7 @@ def main(): for filename, sha256 in BASEBOX_CHECKSUMS[config['basebox_version']][config['vm_provider']].items(): verify_file_sha256(os.path.join(get_vagrant_home(), 'boxes', - BASEBOX_DEFAULT.replace('/', '-VAGRANTSLASH-'), + config['basebox'].replace('/', '-VAGRANTSLASH-'), config['basebox_version'], config['vm_provider'], filename), @@ -375,8 +351,6 @@ def main(): .format(box=config['basebox'], provider=config['vm_provider'], version=config['basebox_version'])) - else: - logging.debug('not updating basebox ...') else: logging.debug('using unverified basebox ...') @@ -389,7 +363,7 @@ def main(): logging.error("'vagrant up' failed.") sys.exit(1) - if config['copy_caches_from_host']: + if options.copy_caches_from_host: ssh_config = v.ssh_config() user = re.search(r'User ([^ \n]+)', ssh_config).group(1) hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)