mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
buildserver: add copy_caches_from_host config option
For people using slow, expensive, and/or flaky internet, liberal use of caching can make a huge difference. The restricted environment of the gpjenkins box has been a good test environment for this (Tor-only, whitelist of allowed IPs to visit, home internet connection).
This commit is contained in:
parent
4cde71552f
commit
daade7656a
@ -28,6 +28,16 @@
|
|||||||
#
|
#
|
||||||
# apt_package_cache = True
|
# apt_package_cache = True
|
||||||
|
|
||||||
|
# The buildserver can use some local caches to speed up builds,
|
||||||
|
# especially when the internet connection is slow and/or expensive.
|
||||||
|
# If enabled, the buildserver setup will look for standard caches in
|
||||||
|
# your HOME dir and copy them to the buildserver VM. Be aware: this
|
||||||
|
# will reduce the isolation of the buildserver from your host machine,
|
||||||
|
# so the buildserver will provide an environment only as trustworthy
|
||||||
|
# as the host machine's environment.
|
||||||
|
#
|
||||||
|
# copy_caches_from_host = True
|
||||||
|
|
||||||
# To specify which Debian mirror the build server VM should use, by
|
# To specify which Debian mirror the build server VM should use, by
|
||||||
# default it uses http.debian.net, which auto-detects which is the
|
# default it uses http.debian.net, which auto-detects which is the
|
||||||
# best mirror to use.
|
# best mirror to use.
|
||||||
|
@ -49,6 +49,7 @@ cd $WORKSPACE
|
|||||||
echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
|
echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
|
||||||
echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
|
echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
|
||||||
echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
|
echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
|
||||||
|
echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py
|
||||||
./makebuildserver --verbose --clean
|
./makebuildserver --verbose --clean
|
||||||
|
|
||||||
# this can be handled in the jenkins job, or here:
|
# this can be handled in the jenkins job, or here:
|
||||||
|
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
import re
|
||||||
import requests
|
import requests
|
||||||
import stat
|
import stat
|
||||||
import sys
|
import sys
|
||||||
@ -39,6 +40,7 @@ config = {
|
|||||||
],
|
],
|
||||||
'debian_mirror': 'http://http.debian.net/debian/',
|
'debian_mirror': 'http://http.debian.net/debian/',
|
||||||
'apt_package_cache': False,
|
'apt_package_cache': False,
|
||||||
|
'copy_caches_from_host': False,
|
||||||
'boot_timeout': 600,
|
'boot_timeout': 600,
|
||||||
'cachedir': cachedir,
|
'cachedir': cachedir,
|
||||||
'cpus': 1,
|
'cpus': 1,
|
||||||
@ -437,6 +439,27 @@ def main():
|
|||||||
print("Configuring build server VM")
|
print("Configuring build server VM")
|
||||||
v.up(provision=True)
|
v.up(provision=True)
|
||||||
|
|
||||||
|
if config['copy_caches_from_host']:
|
||||||
|
ssh_config = v.ssh_config()
|
||||||
|
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
|
||||||
|
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
|
||||||
|
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
|
||||||
|
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
|
||||||
|
|
||||||
|
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
|
||||||
|
fullpath = os.path.join(os.getenv('HOME'), d)
|
||||||
|
if os.path.isdir(fullpath):
|
||||||
|
# TODO newer versions of vagrant provide `vagrant rsync`
|
||||||
|
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
|
||||||
|
subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
|
||||||
|
'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
|
||||||
|
fullpath + '/',
|
||||||
|
user + '@' + hostname + ':~/' + d + '/'])
|
||||||
|
|
||||||
|
# this file changes every time but should not be cached
|
||||||
|
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
|
||||||
|
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
|
||||||
|
|
||||||
print("Writing buildserver ID")
|
print("Writing buildserver ID")
|
||||||
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
||||||
universal_newlines=True)
|
universal_newlines=True)
|
||||||
|
Loading…
Reference in New Issue
Block a user