diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index b6c0183d..7015cf58 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -28,6 +28,16 @@ # # apt_package_cache = True +# The buildserver can use some local caches to speed up builds, +# especially when the internet connection is slow and/or expensive. +# If enabled, the buildserver setup will look for standard caches in +# your HOME dir and copy them to the buildserver VM. Be aware: this +# will reduce the isolation of the buildserver from your host machine, +# so the buildserver will provide an environment only as trustworthy +# as the host machine's environment. +# +# copy_caches_from_host = True + # To specify which Debian mirror the build server VM should use, by # default it uses http.debian.net, which auto-detects which is the # best mirror to use. diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index ae3391a0..a2923ad4 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -49,6 +49,7 @@ cd $WORKSPACE echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py +echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py ./makebuildserver --verbose --clean # this can be handled in the jenkins job, or here: diff --git a/makebuildserver b/makebuildserver index 6578e27c..4985b22f 100755 --- a/makebuildserver +++ b/makebuildserver @@ -2,6 +2,7 @@ import os import pathlib +import re import requests import stat import sys @@ -39,6 +40,7 @@ config = { ], 'debian_mirror': 'http://http.debian.net/debian/', 'apt_package_cache': False, + 'copy_caches_from_host': False, 'boot_timeout': 600, 'cachedir': cachedir, 'cpus': 1, @@ -437,6 +439,27 @@ def main(): print("Configuring build server VM") v.up(provision=True) + if config['copy_caches_from_host']: + ssh_config = v.ssh_config() + user = re.search(r'User ([^ \n]+)', ssh_config).group(1) + hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1) + port = re.search(r'Port ([0-9]+)', ssh_config).group(1) + key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1) + + for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'): + fullpath = os.path.join(os.getenv('HOME'), d) + if os.path.isdir(fullpath): + # TODO newer versions of vagrant provide `vagrant rsync` + run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d]) + subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e', + 'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port), + fullpath + '/', + user + '@' + hostname + ':~/' + d + '/']) + + # this file changes every time but should not be cached + run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock']) + run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/']) + print("Writing buildserver ID") p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, universal_newlines=True)