mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
buildserver: add copy_caches_from_host config option
For people using slow, expensive, and/or flaky internet, liberal use of caching can make a huge difference. The restricted environment of the gpjenkins box has been a good test environment for this (Tor-only, whitelist of allowed IPs to visit, home internet connection).
This commit is contained in:
parent
4cde71552f
commit
daade7656a
@ -28,6 +28,16 @@
|
||||
#
|
||||
# apt_package_cache = True
|
||||
|
||||
# The buildserver can use some local caches to speed up builds,
|
||||
# especially when the internet connection is slow and/or expensive.
|
||||
# If enabled, the buildserver setup will look for standard caches in
|
||||
# your HOME dir and copy them to the buildserver VM. Be aware: this
|
||||
# will reduce the isolation of the buildserver from your host machine,
|
||||
# so the buildserver will provide an environment only as trustworthy
|
||||
# as the host machine's environment.
|
||||
#
|
||||
# copy_caches_from_host = True
|
||||
|
||||
# To specify which Debian mirror the build server VM should use, by
|
||||
# default it uses http.debian.net, which auto-detects which is the
|
||||
# best mirror to use.
|
||||
|
@ -49,6 +49,7 @@ cd $WORKSPACE
|
||||
echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
|
||||
echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
|
||||
echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
|
||||
echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py
|
||||
./makebuildserver --verbose --clean
|
||||
|
||||
# this can be handled in the jenkins job, or here:
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import requests
|
||||
import stat
|
||||
import sys
|
||||
@ -39,6 +40,7 @@ config = {
|
||||
],
|
||||
'debian_mirror': 'http://http.debian.net/debian/',
|
||||
'apt_package_cache': False,
|
||||
'copy_caches_from_host': False,
|
||||
'boot_timeout': 600,
|
||||
'cachedir': cachedir,
|
||||
'cpus': 1,
|
||||
@ -437,6 +439,27 @@ def main():
|
||||
print("Configuring build server VM")
|
||||
v.up(provision=True)
|
||||
|
||||
if config['copy_caches_from_host']:
|
||||
ssh_config = v.ssh_config()
|
||||
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
|
||||
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
|
||||
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
|
||||
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
|
||||
|
||||
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
|
||||
fullpath = os.path.join(os.getenv('HOME'), d)
|
||||
if os.path.isdir(fullpath):
|
||||
# TODO newer versions of vagrant provide `vagrant rsync`
|
||||
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
|
||||
subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
|
||||
'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
|
||||
fullpath + '/',
|
||||
user + '@' + hostname + ':~/' + d + '/'])
|
||||
|
||||
# this file changes every time but should not be cached
|
||||
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
|
||||
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
|
||||
|
||||
print("Writing buildserver ID")
|
||||
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
|
Loading…
Reference in New Issue
Block a user