2016-01-04 16:33:20 +01:00
#!/usr/bin/env python3
2012-09-24 15:04:58 +02:00
import os
2016-06-23 17:12:06 +02:00
import re
2016-06-10 11:42:42 +02:00
import requests
2016-09-06 14:22:29 +02:00
import stat
2012-09-24 15:04:58 +02:00
import sys
2017-04-09 00:57:59 +02:00
import shutil
2012-09-24 15:04:58 +02:00
import subprocess
2016-09-27 08:49:32 +02:00
import vagrant
2014-01-10 19:54:53 +01:00
import hashlib
2016-07-04 20:22:00 +02:00
import yaml
2017-05-22 16:57:47 +02:00
import json
2017-03-24 20:04:50 +01:00
import logging
2016-06-10 11:42:42 +02:00
from clint.textui import progress
2013-10-20 22:16:42 +02:00
from optparse import OptionParser
2016-06-20 11:33:46 +02:00
import fdroidserver.tail
2017-03-26 01:41:39 +01:00
import fdroidserver.vmtools
2013-05-31 08:48:39 +02:00
2014-05-07 16:13:22 +02:00
2013-10-20 22:16:42 +02:00
parser = OptionParser()
2017-03-24 20:04:50 +01:00
parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
2014-05-07 16:13:22 +02:00
help="Spew out even more information than normal")
2017-03-24 20:04:50 +01:00
parser.add_option('-q', action='store_const', const=0, dest='verbosity')
2013-10-20 22:16:42 +02:00
parser.add_option("-c", "--clean", action="store_true", default=False,
2014-05-07 16:13:22 +02:00
help="Build from scratch, rather than attempting to update the existing server")
2017-03-24 20:04:50 +01:00
parser.add_option('--skip-cache-update', action="store_true", default=False,
help="""Skip downloading and checking cache."""
"""This assumes that the cache is already downloaded completely.""")
2017-05-22 17:29:12 +02:00
parser.add_option('--keep-box-file', action="store_true", default=False,
2017-05-22 15:50:32 +02:00
help="""Box file will not be deleted after adding it to box storage"""
""" (KVM-only).""")
2013-10-20 22:16:42 +02:00
options, args = parser.parse_args()
2017-03-26 01:41:39 +01:00
logger = logging.getLogger('fdroidserver-makebuildserver')
2017-03-24 20:04:50 +01:00
if options.verbosity >= 2:
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
logger.setLevel(logging.DEBUG)
elif options.verbosity == 1:
logging.basicConfig(format='%(message)s', level=logging.INFO)
logger.setLevel(logging.INFO)
elif options.verbosity <= 0:
logging.basicConfig(format='%(message)s', level=logging.WARNING)
logger.setLevel(logging.WARNING)
if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
sys.exit(1)
tail = None
2016-01-20 21:00:16 +01:00
# set up default config
2016-01-22 13:33:14 +01:00
cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
2017-03-24 20:04:50 +01:00
logger.debug('cachedir set to: %s', cachedir)
2018-09-25 15:47:33 +02:00
BASEBOX_DEFAULT = 'fdroid/basebox-stretch64'
2019-01-28 17:27:07 +01:00
BASEBOX_VERSION_DEFAULT = '0.5.1'
2018-09-25 15:47:33 +02:00
BASEBOX_CHECKSUMS = {
2019-01-28 17:27:07 +01:00
'0.5.1': {
'libvirt': {
'box.img': 'ad015940b866e36a593ef5fa0035ec6703f74a7f082ab76a1d2bd9463714cd4a',
'metadata.json': '5ced8ecf886722a5152095e099b778b1d2b859c2e1dcf834182274034b8a629d',
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
},
'virtualbox': {
'box-disk1.vmdk': 'cba36a9c9814bdff9aabaea8786c27477ef8958cf6ee65ad844cb2726bdab93e',
'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
}
},
2018-09-25 15:47:33 +02:00
'0.5': {
'virtualbox': {
'box-disk1.vmdk': '8834d5eb78758437c2517f83282172fd5e3842d88f657d577592d0917cd02f89',
'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
},
'libvirt': {
'box.img': '2ef5f1fdc98c24a4f67cecb526d21e1d73dedf5a0072ceff528a0e75da3ff452',
'metadata.json': 'da79a5e2327dcf81a18a9d66a6e91205a20e440f23d3928e633fd39d60c641e5',
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
}
},
'0.3': {
'libvirt': {
'box.img': '24f06f415dde4cdb01d68c904fc57386ea060ba7b94e700670c58694b3d3635e',
'metadata.json': '0965955659082fd2e67723deb3311ba253c96153d3176d856db1b3e6e461cf23',
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
},
'virtualbox': {
'box-disk1.vmdk': '103114977f1a36f7121ef9b3a1495129baa10bfedfada61a13345c8863c4dcd6',
'box.ovf': '33a5fbaf3dba443237baefcba6d56ca7a76121ca530f1140aa8263a69d7d3695',
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
}
}
}
2016-01-20 21:00:16 +01:00
config = {
2018-09-25 15:47:33 +02:00
'basebox': BASEBOX_DEFAULT,
2018-07-29 13:43:48 +02:00
'debian_mirror': 'http://deb.debian.org/debian/',
2016-01-26 11:21:21 +01:00
'apt_package_cache': False,
2016-06-23 17:12:06 +02:00
'copy_caches_from_host': False,
2016-01-26 17:35:25 +01:00
'boot_timeout': 600,
2016-01-22 13:33:14 +01:00
'cachedir': cachedir,
2016-01-20 21:00:16 +01:00
'cpus': 1,
2017-10-20 15:20:51 +02:00
'memory': 2048,
2016-07-04 20:22:00 +02:00
'hwvirtex': 'off',
2016-09-15 11:12:18 +02:00
'vm_provider': 'virtualbox',
2016-01-20 21:00:16 +01:00
}
2016-01-20 20:38:18 +01:00
2016-09-15 11:12:18 +02:00
if os.path.isfile('/usr/bin/systemd-detect-virt'):
try:
virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
2018-05-25 12:32:34 +02:00
except subprocess.CalledProcessError:
2016-09-15 11:12:18 +02:00
virt = 'none'
2016-09-23 17:40:23 +02:00
if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
2017-03-24 20:04:50 +01:00
logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
2016-09-15 11:12:18 +02:00
config['vm_provider'] = 'libvirt'
elif virt != 'none':
2017-03-24 20:04:50 +01:00
logger.info('Running in an unsupported VM guest (%s)!', virt)
2017-04-13 12:49:14 +02:00
logger.debug('detected virt: %s', virt)
2016-09-15 11:12:18 +02:00
2016-01-20 20:38:18 +01:00
# load config file, if present
if os.path.exists('makebuildserver.config.py'):
2016-01-04 17:24:57 +01:00
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
2016-01-20 20:38:18 +01:00
elif os.path.exists('makebs.config.py'):
# this is the old name for the config file
2016-01-04 17:24:57 +01:00
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
2016-07-27 11:20:17 +02:00
if '__builtins__' in config:
del(config['__builtins__']) # added by compile/exec
2017-03-24 20:04:50 +01:00
logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
2018-09-25 15:47:33 +02:00
if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config:
config['basebox_version'] = BASEBOX_VERSION_DEFAULT
# note: vagrant allows putting '/' into the name of a local box,
# so this check is not completely relyable, but better than nothing
if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']:
logger.critical("Can not get version '{version}' for basebox '{box}', "
"vagrant does not support versioning for locally added boxes."
.format(box=config['basebox'], version=config['basebox_version']))
2013-10-20 22:16:42 +02:00
2013-05-25 14:55:16 +02:00
# Update cached files.
2016-01-21 14:09:12 +01:00
cachedir = config['cachedir']
2013-05-25 14:55:16 +02:00
if not os.path.exists(cachedir):
2016-01-04 17:24:57 +01:00
os.makedirs(cachedir, 0o755)
2017-03-24 20:04:50 +01:00
logger.debug('created cachedir %s because it did not exists.', cachedir)
2014-05-31 23:00:27 +02:00
2016-09-23 15:19:15 +02:00
if config['vm_provider'] == 'libvirt':
tmp = cachedir
while tmp != '/':
mode = os.stat(tmp).st_mode
if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
2017-03-24 20:04:50 +01:00
logger.critical('ERROR: %s will not be accessible to the VM! To fix, run:', tmp)
logger.critical(' chmod a+X %s', tmp)
2016-09-23 15:19:15 +02:00
sys.exit(1)
tmp = os.path.dirname(tmp)
2017-03-24 20:04:50 +01:00
logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
2016-09-06 14:22:29 +02:00
2016-07-04 20:22:00 +02:00
if config['apt_package_cache']:
config['aptcachedir'] = cachedir + '/apt/archives'
2017-03-24 20:04:50 +01:00
logger.debug('aptcachedir is set to %s', config['aptcachedir'])
2017-04-09 00:57:59 +02:00
aptcachelock = os.path.join(config['aptcachedir'], 'lock')
if os.path.isfile(aptcachelock):
logger.info('apt cache dir is locked, removing lock')
os.remove(aptcachelock)
aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
if os.path.isdir(aptcachepartial):
logger.info('removing partial downloads from apt cache dir')
shutil.rmtree(aptcachepartial)
2016-07-04 20:22:00 +02:00
2013-05-25 14:55:16 +02:00
cachefiles = [
2017-09-03 19:30:00 +02:00
# Don't update sdk tools beyond 25.2.5.
2017-09-03 18:33:03 +02:00
# Support for android update project has been removed and there is no replacement.
# Until we find a solution for that we need to stay at this revision.
2017-06-09 19:35:04 +02:00
('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
'577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
2017-03-29 12:33:38 +02:00
('https://dl.google.com/android/repository/android_m2repository_r47.zip',
'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.1_r03.zip',
2015-11-20 15:43:47 +01:00
'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.2_r03.zip',
2015-11-20 15:43:47 +01:00
'7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
2015-11-20 15:43:47 +01:00
'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
2015-11-20 15:43:47 +01:00
'54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-3.0_r02.zip',
2015-11-20 15:43:47 +01:00
'1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-3.1_r03.zip',
2015-11-20 15:43:47 +01:00
'7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-3.2_r01.zip',
2015-11-20 15:43:47 +01:00
'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-14_r04.zip',
2015-11-20 15:43:47 +01:00
'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-15_r05.zip',
2015-11-20 15:43:47 +01:00
'5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-16_r05.zip',
2015-11-20 15:43:47 +01:00
'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-17_r03.zip',
2015-11-20 15:43:47 +01:00
'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-18_r03.zip',
2015-11-20 15:43:47 +01:00
'166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-19_r04.zip',
2015-11-20 15:43:47 +01:00
'5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-20_r02.zip',
2015-11-20 15:43:47 +01:00
'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-21_r02.zip',
2015-11-20 15:43:47 +01:00
'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-22_r02.zip',
2015-11-20 15:43:47 +01:00
'45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/platform-23_r03.zip',
2016-05-26 17:05:39 +02:00
'4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
2016-08-30 19:47:37 +02:00
('https://dl.google.com/android/repository/platform-24_r02.zip',
'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
2017-08-17 00:41:23 +02:00
('https://dl.google.com/android/repository/platform-25_r03.zip',
'9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
('https://dl.google.com/android/repository/platform-26_r02.zip',
'2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
2018-05-24 00:27:40 +02:00
('https://dl.google.com/android/repository/platform-27_r03.zip',
'020c4c090bc82ce87ebaae5d1a922e21b39a1d03c78ffa43f0c3e42fc7d28169'),
2018-11-28 18:51:53 +01:00
('https://dl.google.com/android/repository/platform-28_r06.zip',
'8452dbbf9668a428abb243c4f02a943b7aa83af3cca627629a15c4c09f28e7bd'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
2015-11-20 21:14:08 +01:00
'3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
2015-11-20 21:14:08 +01:00
'296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
2015-11-20 21:14:08 +01:00
'3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
2015-11-20 21:14:08 +01:00
'91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
2015-11-20 21:14:08 +01:00
'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
2015-11-20 21:14:08 +01:00
'82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
2016-04-05 11:49:30 +02:00
'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
2016-06-24 17:54:30 +02:00
('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
2016-07-25 12:40:44 +02:00
('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
2016-08-30 18:19:34 +02:00
'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
2016-08-30 18:19:49 +02:00
('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
'924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
2016-10-03 10:10:41 +02:00
('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
2016-10-23 17:47:19 +02:00
('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
'74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
2016-12-01 08:38:31 +01:00
('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
'671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
2016-12-17 23:36:48 +01:00
('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
'1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
2017-05-26 16:54:06 +02:00
('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
'152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
2017-06-09 19:35:04 +02:00
('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
'7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
2017-08-17 00:41:23 +02:00
('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
2017-10-04 19:03:41 +02:00
('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
2017-11-12 08:48:00 +01:00
('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
'5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
2017-11-05 08:59:01 +01:00
('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
'53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
2017-11-10 16:38:58 +01:00
('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
'2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
2017-12-15 12:34:40 +01:00
('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
2018-01-25 11:53:27 +01:00
('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
'5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
2018-06-26 15:19:17 +02:00
('https://dl.google.com/android/repository/build-tools_r28-linux.zip',
'a7c6b73661836a6c50e32c06e7b8e8d6ef8c96f6812f269b9c90470dfb37753c'),
2018-07-25 17:20:25 +02:00
('https://dl.google.com/android/repository/build-tools_r28.0.1-linux.zip',
'4825b52a6258a9c2ab073720a5e91e2fa81b552b48024ee81887b863397416af'),
2018-08-10 20:54:02 +02:00
('https://dl.google.com/android/repository/build-tools_r28.0.2-linux.zip',
'12cebcafd8f30119c7ef53ffb3562a5b5b9f776c9399038587c18df44ea6452b'),
2018-12-13 22:00:15 +01:00
('https://dl.google.com/android/repository/build-tools_r28.0.3-linux.zip',
'7954956a40633c88f693d638cbc23f68e9e2499dc7a4b7dfdaf6a3e91387749a'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
2014-12-12 12:29:16 +01:00
'420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
2015-04-09 16:02:35 +02:00
'010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
2015-06-18 16:57:00 +02:00
'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
2015-08-13 20:28:01 +02:00
'3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
2015-08-18 22:40:21 +02:00
'18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
2015-10-08 15:16:16 +02:00
'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
2015-10-25 10:48:17 +01:00
'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
2015-11-20 21:24:07 +01:00
'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
2016-01-05 09:15:54 +01:00
'66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
2016-02-15 12:44:07 +01:00
'8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
2016-04-05 11:49:15 +02:00
'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
2016-06-10 11:42:42 +02:00
('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
2016-05-26 16:45:54 +02:00
'0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
2016-06-24 18:03:48 +02:00
('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
'993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
2016-08-12 19:52:32 +02:00
('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
2016-08-18 17:53:27 +02:00
('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
'39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
2016-09-23 07:50:52 +02:00
('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
2016-11-15 21:56:15 +01:00
('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
'5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
2016-12-02 06:51:38 +01:00
('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
'9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
2017-01-08 21:13:24 +01:00
('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
2017-02-26 03:23:49 +01:00
('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
'72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
2017-03-10 14:29:36 +01:00
('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
2017-04-15 15:08:13 +02:00
('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
'0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
2017-08-07 13:46:08 +02:00
('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
'8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
2017-06-17 01:39:44 +02:00
('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
'56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
2017-08-07 13:46:08 +02:00
('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
'79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
2017-09-23 09:04:39 +02:00
('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
'515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
2017-10-08 20:10:43 +02:00
('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
2017-10-31 15:19:57 +01:00
('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
'8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
2017-11-14 08:36:12 +01:00
('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
'15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
2017-12-15 12:32:34 +01:00
('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
2018-01-10 19:31:59 +01:00
('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
2018-01-29 17:55:48 +01:00
('https://downloads.gradle.org/distributions/gradle-4.5-bin.zip',
'03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3'),
2018-02-11 08:08:06 +01:00
('https://downloads.gradle.org/distributions/gradle-4.5.1-bin.zip',
'3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e'),
2018-03-16 05:39:44 +01:00
('https://downloads.gradle.org/distributions/gradle-4.6-bin.zip',
'98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915'),
2018-04-20 17:55:51 +02:00
('https://downloads.gradle.org/distributions/gradle-4.7-bin.zip',
'fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04'),
2018-06-14 19:37:54 +02:00
('https://downloads.gradle.org/distributions/gradle-4.8-bin.zip',
'f3e29692a8faa94eb0b02ebf36fa263a642b3ae8694ef806c45c345b8683f1ba'),
2018-06-26 15:19:47 +02:00
('https://downloads.gradle.org/distributions/gradle-4.8.1-bin.zip',
'af334d994b5e69e439ab55b5d2b7d086da5ea6763d78054f49f147b06370ed71'),
2018-07-30 22:54:09 +02:00
('https://downloads.gradle.org/distributions/gradle-4.9-bin.zip',
'e66e69dce8173dd2004b39ba93586a184628bc6c28461bc771d6835f7f9b0d28'),
2018-09-03 14:44:33 +02:00
('https://downloads.gradle.org/distributions/gradle-4.10-bin.zip',
'248cfd92104ce12c5431ddb8309cf713fe58de8e330c63176543320022f59f18'),
2018-09-19 00:03:01 +02:00
('https://downloads.gradle.org/distributions/gradle-4.10.1-bin.zip',
'e53ce3a01cf016b5d294eef20977ad4e3c13e761ac1e475f1ffad4c6141a92bd'),
2018-09-21 06:38:27 +02:00
('https://downloads.gradle.org/distributions/gradle-4.10.2-bin.zip',
'b49c6da1b2cb67a0caf6c7480630b51c70a11ca2016ff2f555eaeda863143a29'),
2018-12-23 18:07:20 +01:00
('https://downloads.gradle.org/distributions/gradle-4.10.3-bin.zip',
'8626cbf206b4e201ade7b87779090690447054bc93f052954c78480fa6ed186e'),
2018-11-27 16:30:29 +01:00
('https://downloads.gradle.org/distributions/gradle-5.0-bin.zip',
'6157ac9f3410bc63644625b3b3e9e96c963afd7910ae0697792db57813ee79a6'),
2019-01-16 05:20:07 +01:00
('https://downloads.gradle.org/distributions/gradle-5.1-bin.zip',
'7506638a380092a0406364c79d6c87d03d23017fc25a5770379d1ce23c3fcd4d'),
('https://downloads.gradle.org/distributions/gradle-5.1.1-bin.zip',
'4953323605c5d7b89e97d0dc7779e275bccedefcdac090aec123375eae0cc798'),
2016-07-04 17:14:06 +02:00
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
2016-08-12 20:24:54 +02:00
('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
2016-08-02 11:09:32 +02:00
('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
2016-11-25 21:10:44 +01:00
('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
2017-05-26 16:54:06 +02:00
('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
'0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
2017-08-11 09:19:40 +02:00
('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
2018-02-11 08:20:42 +01:00
('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip',
'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'),
2018-06-26 15:20:04 +02:00
('https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip',
'5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd'),
2018-10-22 18:31:57 +02:00
('https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip',
'4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd'),
2019-01-19 23:51:46 +01:00
('https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip',
'c0a2425206191252197b97ea5fcc7eab9f693a576e69ef4773a9ed1690feed53'),
2014-12-31 16:42:26 +01:00
]
2014-05-31 23:00:27 +02:00
2014-05-07 16:13:22 +02:00
2014-01-10 19:54:53 +01:00
def sha256_for_file(path):
2016-03-13 21:49:38 +01:00
with open(path, 'rb') as f:
2014-01-10 19:54:53 +01:00
s = hashlib.sha256()
while True:
data = f.read(4096)
if not data:
break
s.update(data)
return s.hexdigest()
2017-03-14 20:14:06 +01:00
2018-09-25 15:47:33 +02:00
def verify_file_sha256(path, sha256):
if sha256_for_file(path) != sha256:
logger.critical("File verification for '{path}' failed! "
"expected sha256 checksum: {checksum}"
.format(path=path, checksum=sha256))
sys.exit(1)
else:
logger.debug("sucessfully verifyed file '{path}' "
"('{checksum}')".format(path=path,
checksum=sha256))
def get_vagrant_home():
return os.environ.get('VAGRANT_HOME',
os.path.join(os.path.expanduser('~'),
'.vagrant.d'))
2016-06-23 16:38:39 +02:00
def run_via_vagrant_ssh(v, cmdlist):
if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
cmd = cmdlist
else:
cmd = ' '.join(cmdlist)
v._run_vagrant_command(['ssh', '-c', cmd])
2017-03-24 20:04:50 +01:00
def update_cache(cachedir, cachefiles):
2016-09-25 18:55:29 +02:00
for srcurl, shasum in cachefiles:
filename = os.path.basename(srcurl)
local_filename = os.path.join(cachedir, filename)
2016-06-10 11:42:42 +02:00
2016-09-25 18:55:29 +02:00
if os.path.exists(local_filename):
2019-01-28 22:16:48 +01:00
if sha256_for_file(local_filename) == shasum:
logger.info("\t...shasum verified for %s", local_filename)
continue
2016-09-25 18:55:29 +02:00
local_length = os.path.getsize(local_filename)
else:
local_length = -1
resume_header = {}
download = True
try:
r = requests.head(srcurl, allow_redirects=True, timeout=60)
if r.status_code == 200:
content_length = int(r.headers.get('content-length'))
else:
content_length = local_length # skip the download
except requests.exceptions.RequestException as e:
content_length = local_length # skip the download
2017-03-24 20:04:50 +01:00
logger.warn('%s', e)
2016-09-25 18:55:29 +02:00
if local_length == content_length:
download = False
elif local_length > content_length:
2017-03-24 20:04:50 +01:00
logger.info('deleting corrupt file from cache: %s', local_filename)
2016-09-25 18:55:29 +02:00
os.remove(local_filename)
2017-03-24 20:04:50 +01:00
logger.info("Downloading %s to cache", filename)
2016-09-25 18:55:29 +02:00
elif local_length > -1 and local_length < content_length:
2017-03-24 20:04:50 +01:00
logger.info("Resuming download of %s", local_filename)
2016-09-25 18:55:29 +02:00
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
else:
2017-03-24 20:04:50 +01:00
logger.info("Downloading %s to cache", filename)
2016-06-10 11:42:42 +02:00
2016-09-25 18:55:29 +02:00
if download:
r = requests.get(srcurl, headers=resume_header,
2017-08-17 00:49:53 +02:00
stream=True, allow_redirects=True)
2016-06-10 11:42:42 +02:00
content_length = int(r.headers.get('content-length'))
2016-09-25 18:55:29 +02:00
with open(local_filename, 'ab') as f:
for chunk in progress.bar(r.iter_content(chunk_size=65536),
expected_size=(content_length / 65536) + 1):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
v = sha256_for_file(local_filename)
if v == shasum:
2017-03-24 20:04:50 +01:00
logger.info("\t...shasum verified for %s", local_filename)
2013-08-09 20:37:13 +02:00
else:
2017-03-24 20:04:50 +01:00
logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
2016-09-25 18:55:29 +02:00
os.remove(local_filename)
sys.exit(1)
2016-07-04 20:22:00 +02:00
2017-03-24 20:04:50 +01:00
2017-04-13 12:49:14 +02:00
def debug_log_vagrant_vm(vm_dir, config):
2017-03-24 20:04:50 +01:00
if options.verbosity >= 3:
_vagrant_dir = os.path.join(vm_dir, '.vagrant')
logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
logger.debug('> vagrant status')
subprocess.call(['vagrant', 'status'], cwd=vm_dir)
logger.debug('> vagrant box list')
subprocess.call(['vagrant', 'box', 'list'])
2017-04-13 12:49:14 +02:00
if config['vm_provider'] == 'libvirt':
logger.debug('> virsh -c qmeu:///system list --all')
subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
2017-05-23 11:59:53 +02:00
domain = 'buildserver_default'
logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
2017-03-24 20:04:50 +01:00
def main():
global cachedir, cachefiles, config, tail
if options.skip_cache_update:
logger.info('skipping cache update and verification...')
else:
update_cache(cachedir, cachefiles)
2016-09-25 18:55:29 +02:00
# use VirtualBox software virtualization if hardware is not available,
# like if this is being run in kvm or some other VM platform, like
# http://jenkins.debian.net, the values are 'on' or 'off'
if sys.platform.startswith('darwin'):
# all < 10 year old Macs work, and OSX servers as VM host are very
# rare, but this could also be auto-detected if someone codes it
config['hwvirtex'] = 'on'
2017-03-24 20:04:50 +01:00
logger.info('platform is darwnin -> hwvirtex = \'on\'')
2016-09-25 18:55:29 +02:00
elif os.path.exists('/proc/cpuinfo'):
with open('/proc/cpuinfo') as f:
contents = f.read()
if 'vmx' in contents or 'svm' in contents:
config['hwvirtex'] = 'on'
2017-03-24 20:04:50 +01:00
logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
2016-09-25 18:55:29 +02:00
serverdir = os.path.join(os.getcwd(), 'buildserver')
logfilename = os.path.join(serverdir, 'up.log')
if not os.path.exists(logfilename):
open(logfilename, 'a').close() # create blank file
log_cm = vagrant.make_file_cm(logfilename)
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
2017-03-24 20:04:50 +01:00
if options.verbosity >= 2:
2016-09-25 18:55:29 +02:00
tail = fdroidserver.tail.Tail(logfilename)
tail.start()
2017-03-26 03:15:33 +02:00
vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
2016-09-25 18:55:29 +02:00
if options.clean:
2017-03-26 01:41:39 +01:00
vm.destroy()
2016-09-25 18:55:29 +02:00
# Check against the existing Vagrantfile.yaml, and if they differ, we
# need to create a new box:
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
writevf = True
if os.path.exists(vf):
2017-03-24 20:04:50 +01:00
logger.info('Halting %s', serverdir)
2016-09-25 18:55:29 +02:00
v.halt()
with open(vf, 'r', encoding='utf-8') as f:
oldconfig = yaml.load(f)
if config != oldconfig:
2017-03-24 20:04:50 +01:00
logger.info("Server configuration has changed, rebuild from scratch is required")
2017-03-26 01:41:39 +01:00
vm.destroy()
2016-09-25 18:55:29 +02:00
else:
2017-03-24 20:04:50 +01:00
logger.info("Re-provisioning existing server")
2016-09-25 18:55:29 +02:00
writevf = False
else:
2017-03-24 20:04:50 +01:00
logger.info("No existing server - building from scratch")
2016-09-25 18:55:29 +02:00
if writevf:
with open(vf, 'w', encoding='utf-8') as f:
yaml.dump(config, f)
2016-06-20 11:33:46 +02:00
2018-07-28 16:48:15 +02:00
# Check if selected provider is supported
if config['vm_provider'] not in ['libvirt', 'virtualbox']:
logger.critical("Currently selected VM provider '{vm_provider}' "
"is not supported. (please choose from: "
"virtualbox, libvirt)"
.format(vm_provider=config['cm_provider']))
sys.exit(1)
# Check if selected basebox is available
available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
2018-09-13 11:08:51 +02:00
if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
2018-07-28 16:48:15 +02:00
logger.critical("Vagrant box '{basebox}' not available "
"for '{vm_provider}' VM provider. "
"Please make sure it's added to vagrant. "
"(If you need a basebox to begin with, "
"here is how we're bootstrapping it: "
"https://gitlab.com/fdroid/basebox)"
.format(vm_provider=config['vm_provider'],
basebox=config['basebox']))
sys.exit(1)
2016-09-25 18:55:29 +02:00
2018-09-25 15:47:33 +02:00
# download and verfiy fdroid pre-built basebox
if config['basebox'] == BASEBOX_DEFAULT:
buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default'])
if buildserver_not_created or options.clean:
# make vagrant download and add basebox
target_basebox_installed = any([x for x in v.box_list() if x.name == BASEBOX_DEFAULT and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
if not target_basebox_installed:
cmd = [shutil.which('vagrant'), 'box', 'add', BASEBOX_DEFAULT,
'--box-version=' + config['basebox_version'],
'--provider=' + config['vm_provider']]
ret_val = subprocess.call(cmd)
if ret_val != 0:
logger.critical("downloading basebox '{box}' "
"({provider}, version {version}) failed."
.format(box=config['basebox'],
provider=config['vm_provider'],
version=config['basebox_version']))
sys.exit(1)
# verify box
if config['basebox_version'] not in BASEBOX_CHECKSUMS.keys():
logger.critical("can not verify '{box}', "
"unknown basebox version '{version}'"
.format(box=config['basebox'],
version=config['basebox_version']))
sys.exit(1)
for filename, sha256 in BASEBOX_CHECKSUMS[config['basebox_version']][config['vm_provider']].items():
verify_file_sha256(os.path.join(get_vagrant_home(),
'boxes',
BASEBOX_DEFAULT.replace('/', '-VAGRANTSLASH-'),
config['basebox_version'],
config['vm_provider'],
filename),
sha256)
logger.info("successfully verified: '{box}' "
"({provider}, version {version})"
.format(box=config['basebox'],
provider=config['vm_provider'],
version=config['basebox_version']))
else:
logger.debug('not updating basebox ...')
else:
logger.debug('using unverified basebox ...')
2017-03-24 20:04:50 +01:00
logger.info("Configuring build server VM")
2017-04-13 12:49:14 +02:00
debug_log_vagrant_vm(serverdir, config)
2017-03-25 01:56:20 +01:00
try:
2017-05-22 17:12:34 +02:00
v.up(provision=True)
2017-10-20 04:38:39 +02:00
except subprocess.CalledProcessError:
2017-04-13 12:49:14 +02:00
debug_log_vagrant_vm(serverdir, config)
2018-07-28 16:48:15 +02:00
logger.error("'vagrant up' failed.")
2017-03-25 01:56:20 +01:00
sys.exit(1)
2016-09-25 18:55:29 +02:00
2016-06-23 17:12:06 +02:00
if config['copy_caches_from_host']:
ssh_config = v.ssh_config()
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
fullpath = os.path.join(os.getenv('HOME'), d)
if os.path.isdir(fullpath):
2017-12-07 11:52:10 +01:00
ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
'-o StrictHostKeyChecking=no',
'-o UserKnownHostsFile=/dev/null',
'-o LogLevel=FATAL',
'-o IdentitiesOnly=yes',
'-o PasswordAuthentication=no'))
2017-12-06 09:47:08 +01:00
# TODO vagrant 1.5+ provides `vagrant rsync`
2016-06-23 17:12:06 +02:00
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
2017-12-07 15:56:09 +01:00
subprocess.call(['rsync', '-ax', '--delete', '-e',
2017-12-06 09:47:08 +01:00
ssh_command,
2016-06-23 17:12:06 +02:00
fullpath + '/',
user + '@' + hostname + ':~/' + d + '/'])
# this file changes every time but should not be cached
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
2016-09-25 18:55:29 +02:00
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
universal_newlines=True)
buildserverid = p.communicate()[0].strip()
2017-03-24 20:04:50 +01:00
logger.info("Writing buildserver ID ...ID is %s", buildserverid)
2016-06-23 16:38:39 +02:00
run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
2016-09-25 18:55:29 +02:00
2017-03-24 20:04:50 +01:00
logger.info("Stopping build server VM")
2016-09-27 08:49:32 +02:00
v.halt()
2013-05-31 08:48:39 +02:00
2017-03-24 20:04:50 +01:00
logger.info("Packaging")
2016-09-26 11:22:05 +02:00
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
if os.path.exists(boxfile):
os.remove(boxfile)
2017-02-14 22:58:35 +01:00
2017-05-22 15:50:32 +02:00
vm.package(output=boxfile)
2016-09-25 18:55:29 +02:00
2017-03-24 20:04:50 +01:00
logger.info("Adding box")
2017-05-22 17:14:48 +02:00
vm.box_add('buildserver', boxfile, force=True)
2016-09-25 18:55:29 +02:00
2017-05-22 17:14:48 +02:00
if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
2017-05-22 17:12:34 +02:00
logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
sys.exit(1)
2017-05-22 17:29:12 +02:00
if not options.keep_box_file:
2018-05-29 12:31:56 +02:00
logger.debug("""box added to vagrant, removing generated box file '%s'""",
2017-05-22 17:29:12 +02:00
boxfile)
os.remove(boxfile)
2016-09-25 18:55:29 +02:00
if __name__ == '__main__':
try:
main()
finally:
if tail is not None:
tail.stop()
