2016-01-04 16:33:20 +01:00
#!/usr/bin/env python3
2012-09-24 15:04:58 +02:00
import os
2016-06-23 17:12:06 +02:00
import re
2016-06-10 11:42:42 +02:00
import requests
2016-09-06 14:22:29 +02:00
import stat
2012-09-24 15:04:58 +02:00
import sys
2017-04-09 00:57:59 +02:00
import shutil
2012-09-24 15:04:58 +02:00
import subprocess
2016-09-27 08:49:32 +02:00
import vagrant
2014-01-10 19:54:53 +01:00
import hashlib
2016-07-04 20:22:00 +02:00
import yaml
2017-05-22 16:57:47 +02:00
import json
2017-03-24 20:04:50 +01:00
import logging
2016-06-10 11:42:42 +02:00
from clint.textui import progress
2013-10-20 22:16:42 +02:00
from optparse import OptionParser
2016-06-20 11:33:46 +02:00
import fdroidserver.tail
2017-03-26 01:41:39 +01:00
import fdroidserver.vmtools
2013-05-31 08:48:39 +02:00
2014-05-07 16:13:22 +02:00
2013-10-20 22:16:42 +02:00
parser = OptionParser()
2017-03-24 20:04:50 +01:00
parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
2014-05-07 16:13:22 +02:00
help="Spew out even more information than normal")
2017-03-24 20:04:50 +01:00
parser.add_option('-q', action='store_const', const=0, dest='verbosity')
2013-10-20 22:16:42 +02:00
parser.add_option("-c", "--clean", action="store_true", default=False,
2014-05-07 16:13:22 +02:00
help="Build from scratch, rather than attempting to update the existing server")
2017-03-24 20:04:50 +01:00
parser.add_option('--skip-cache-update', action="store_true", default=False,
help="""Skip downloading and checking cache."""
"""This assumes that the cache is already downloaded completely.""")
2017-05-22 17:29:12 +02:00
parser.add_option('--keep-box-file', action="store_true", default=False,
2017-05-22 15:50:32 +02:00
help="""Box file will not be deleted after adding it to box storage"""
""" (KVM-only).""")
2013-10-20 22:16:42 +02:00
options, args = parser.parse_args()
2019-02-20 15:54:32 +01:00
logformat = '%(levelname)s: %(message)s'
loglevel = logging.DEBUG
if options.verbosity == 1:
loglevel = logging.INFO
2017-03-24 20:04:50 +01:00
elif options.verbosity <= 0:
2019-02-20 15:54:32 +01:00
loglevel = logging.WARNING
logging.basicConfig(format=logformat, level=loglevel)
2017-03-24 20:04:50 +01:00
tail = None
2021-09-03 12:18:13 +02:00
BASEBOX_DEFAULT = 'fdroid/bullseye64'
BASEBOX_VERSION_DEFAULT = '11.20220317.1'
2018-09-25 15:47:33 +02:00
BASEBOX_CHECKSUMS = {
2021-09-03 12:18:13 +02:00
'11.20220317.1': {
2021-05-27 15:54:45 +02:00
'libvirt': {
2021-09-03 12:18:13 +02:00
'box.img': 'fbde152a2f61d191983be9d1dbeae2591af32cca1ec27daa342485d97187515e',
'metadata.json': '42b96a01106c25f3a222ddad0baead0b811cc64926f924fb836bbfa43580e646',
'Vagrantfile': 'f9c6fcbb47a4d0d33eb066859c8e87efd642287a638bd7da69a9e7a6f25fec47',
2021-05-27 15:54:45 +02:00
},
'virtualbox': {
2021-09-03 12:18:13 +02:00
'box.ovf': 'becd5cea2666d42e12def13a91766aa0d4b0e8e6f53102486c2a6cdb4e401b08',
'box.vmdk': '49c96a58a3ee99681d348075864a290c60a8d334fddd21be453c825fcee75eda',
'metadata.json': 'ffdaa989f2f6932cd8042e1102371f405cc7ad38e324210a1326192e4689e83a',
'Vagrantfile': '0bbc2ae97668d8da27ab97b766752dcd0bf9e41900e21057de15a58ee7fae47d',
2021-05-27 15:54:45 +02:00
}
},
2018-09-25 15:47:33 +02:00
}
2016-01-20 21:00:16 +01:00
config = {
2018-09-25 15:47:33 +02:00
'basebox': BASEBOX_DEFAULT,
2018-07-29 13:43:48 +02:00
'debian_mirror': 'http://deb.debian.org/debian/',
2016-01-26 11:21:21 +01:00
'apt_package_cache': False,
2016-06-23 17:12:06 +02:00
'copy_caches_from_host': False,
2016-01-26 17:35:25 +01:00
'boot_timeout': 600,
2019-02-20 16:26:57 +01:00
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
2016-01-20 21:00:16 +01:00
'cpus': 1,
2017-10-20 15:20:51 +02:00
'memory': 2048,
2016-07-04 20:22:00 +02:00
'hwvirtex': 'off',
2016-09-15 11:12:18 +02:00
'vm_provider': 'virtualbox',
2016-01-20 21:00:16 +01:00
}
2016-01-20 20:38:18 +01:00
# load config file, if present
if os.path.exists('makebuildserver.config.py'):
2016-01-04 17:24:57 +01:00
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
2016-01-20 20:38:18 +01:00
elif os.path.exists('makebs.config.py'):
# this is the old name for the config file
2016-01-04 17:24:57 +01:00
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
2016-07-27 11:20:17 +02:00
if '__builtins__' in config:
2022-07-31 09:14:17 +02:00
del config['__builtins__'] # added by compile/exec
2019-02-20 15:54:32 +01:00
logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
2018-09-25 15:47:33 +02:00
if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config:
config['basebox_version'] = BASEBOX_VERSION_DEFAULT
# note: vagrant allows putting '/' into the name of a local box,
2021-09-03 12:18:13 +02:00
# so this check is not completely reliable, but better than nothing
2018-09-25 15:47:33 +02:00
if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']:
2019-02-20 15:54:32 +01:00
logging.critical("Can not get version '{version}' for basebox '{box}', "
"vagrant does not support versioning for locally added boxes."
.format(box=config['basebox'], version=config['basebox_version']))
2013-10-20 22:16:42 +02:00
2013-05-25 14:55:16 +02:00
# Update cached files.
2019-02-20 16:26:57 +01:00
if not os.path.exists(config['cachedir']):
os.makedirs(config['cachedir'], 0o755)
logging.debug('created cachedir {} because it did not exists.'.format(config['cachedir']))
2014-05-31 23:00:27 +02:00
2016-09-23 15:19:15 +02:00
if config['vm_provider'] == 'libvirt':
2019-02-20 16:26:57 +01:00
tmp = config['cachedir']
2016-09-23 15:19:15 +02:00
while tmp != '/':
mode = os.stat(tmp).st_mode
if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
2019-02-20 15:54:32 +01:00
logging.critical('ERROR: %s will not be accessible to the VM! To fix, run:', tmp)
logging.critical(' chmod a+X %s', tmp)
2016-09-23 15:19:15 +02:00
sys.exit(1)
tmp = os.path.dirname(tmp)
2019-02-20 16:26:57 +01:00
logging.debug('cache dir %s is accessible for libvirt vm.', config['cachedir'])
2016-09-06 14:22:29 +02:00
2016-07-04 20:22:00 +02:00
if config['apt_package_cache']:
2019-02-20 16:26:57 +01:00
config['aptcachedir'] = config['cachedir'] + '/apt/archives'
2019-02-20 15:54:32 +01:00
logging.debug('aptcachedir is set to %s', config['aptcachedir'])
2017-04-09 00:57:59 +02:00
aptcachelock = os.path.join(config['aptcachedir'], 'lock')
if os.path.isfile(aptcachelock):
2019-02-20 15:54:32 +01:00
logging.info('apt cache dir is locked, removing lock')
2017-04-09 00:57:59 +02:00
os.remove(aptcachelock)
aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
if os.path.isdir(aptcachepartial):
2019-02-20 15:54:32 +01:00
logging.info('removing partial downloads from apt cache dir')
2017-04-09 00:57:59 +02:00
shutil.rmtree(aptcachepartial)
2016-07-04 20:22:00 +02:00
2019-02-20 15:47:43 +01:00
CACHE_FILES = [
2017-09-03 19:30:00 +02:00
# Don't update sdk tools beyond 25.2.5.
2017-09-03 18:33:03 +02:00
# Support for android update project has been removed and there is no replacement.
# Until we find a solution for that we need to stay at this revision.
2017-06-09 19:35:04 +02:00
('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
'577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
2017-03-29 12:33:38 +02:00
('https://dl.google.com/android/repository/android_m2repository_r47.zip',
'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.1_r03.zip',
2015-11-20 15:43:47 +01:00
'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.2_r03.zip',
2015-11-20 15:43:47 +01:00
'7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
2015-11-20 15:43:47 +01:00
'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
2015-11-20 15:43:47 +01:00
'54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-3.0_r02.zip',
2015-11-20 15:43:47 +01:00
'1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-3.1_r03.zip',
2015-11-20 15:43:47 +01:00
'7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
2016-06-14 16:34:13 +02:00
('https://dl.google.com/android/repository/android-3.2_r01.zip',
2015-11-20 15:43:47 +01:00
'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-14_r04.zip',
2015-11-20 15:43:47 +01:00
'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-15_r05.zip',
2015-11-20 15:43:47 +01:00
'5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-16_r05.zip',
2015-11-20 15:43:47 +01:00
'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-17_r03.zip',
2015-11-20 15:43:47 +01:00
'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-18_r03.zip',
2015-11-20 15:43:47 +01:00
'166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-19_r04.zip',
2015-11-20 15:43:47 +01:00
'5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-20_r02.zip',
2015-11-20 15:43:47 +01:00
'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-21_r02.zip',
2015-11-20 15:43:47 +01:00
'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/android-22_r02.zip',
2015-11-20 15:43:47 +01:00
'45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/platform-23_r03.zip',
2016-05-26 17:05:39 +02:00
'4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
2016-08-30 19:47:37 +02:00
('https://dl.google.com/android/repository/platform-24_r02.zip',
'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
2017-08-17 00:41:23 +02:00
('https://dl.google.com/android/repository/platform-25_r03.zip',
'9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
('https://dl.google.com/android/repository/platform-26_r02.zip',
'2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
2018-05-24 00:27:40 +02:00
('https://dl.google.com/android/repository/platform-27_r03.zip',
'020c4c090bc82ce87ebaae5d1a922e21b39a1d03c78ffa43f0c3e42fc7d28169'),
2018-11-28 18:51:53 +01:00
('https://dl.google.com/android/repository/platform-28_r06.zip',
'8452dbbf9668a428abb243c4f02a943b7aa83af3cca627629a15c4c09f28e7bd'),
2020-08-10 02:06:30 +02:00
('https://dl.google.com/android/repository/platform-29_r05.zip',
'951da8bf175254da74626824f919bd28def64f8828f29dd3b124a535cf4049d8'),
2020-11-09 09:24:39 +01:00
('https://dl.google.com/android/repository/platform-30_r03.zip',
'f3f5b75744dbf6ee6ed3e8174a71e513bfee502d0bc3463ea97e517bff68d84e'),
2021-07-22 13:26:32 +02:00
('https://dl.google.com/android/repository/platform-31_r01.zip',
'1d69fe1d7f9788d82ff3a374faf4f6ccc9d1d372aa84a86b5bcfb517523b0b3f'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
2015-11-20 21:14:08 +01:00
'3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
2015-11-20 21:14:08 +01:00
'296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
2015-11-20 21:14:08 +01:00
'3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
2015-11-20 21:14:08 +01:00
'91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
2015-11-20 21:14:08 +01:00
'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
2015-11-20 21:14:08 +01:00
'82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
2016-06-10 11:42:42 +02:00
('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
2016-04-05 11:49:30 +02:00
'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
2016-06-24 17:54:30 +02:00
('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
2016-07-25 12:40:44 +02:00
('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
2016-08-30 18:19:34 +02:00
'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
2016-08-30 18:19:49 +02:00
('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
'924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
2016-10-03 10:10:41 +02:00
('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
2016-10-23 17:47:19 +02:00
('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
'74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
2016-12-01 08:38:31 +01:00
('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
'671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
2016-12-17 23:36:48 +01:00
('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
'1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
2017-05-26 16:54:06 +02:00
('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
'152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
2017-06-09 19:35:04 +02:00
('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
'7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
2017-08-17 00:41:23 +02:00
('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
2017-10-04 19:03:41 +02:00
('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
2017-11-12 08:48:00 +01:00
('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
'5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
2017-11-05 08:59:01 +01:00
('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
'53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
2017-11-10 16:38:58 +01:00
('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
'2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
2017-12-15 12:34:40 +01:00
('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
2018-01-25 11:53:27 +01:00
('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
'5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
2018-06-26 15:19:17 +02:00
('https://dl.google.com/android/repository/build-tools_r28-linux.zip',
'a7c6b73661836a6c50e32c06e7b8e8d6ef8c96f6812f269b9c90470dfb37753c'),
2018-07-25 17:20:25 +02:00
('https://dl.google.com/android/repository/build-tools_r28.0.1-linux.zip',
'4825b52a6258a9c2ab073720a5e91e2fa81b552b48024ee81887b863397416af'),
2018-08-10 20:54:02 +02:00
('https://dl.google.com/android/repository/build-tools_r28.0.2-linux.zip',
'12cebcafd8f30119c7ef53ffb3562a5b5b9f776c9399038587c18df44ea6452b'),
2018-12-13 22:00:15 +01:00
('https://dl.google.com/android/repository/build-tools_r28.0.3-linux.zip',
'7954956a40633c88f693d638cbc23f68e9e2499dc7a4b7dfdaf6a3e91387749a'),
2019-08-26 11:46:35 +02:00
('https://dl.google.com/android/repository/build-tools_r29.0.2-linux.zip',
'1e9393cbfd4a4b82e30e7f55ab38db4a5a3259db93d5821c63597bc74522fa08'),
2020-03-17 12:01:41 +01:00
('https://dl.google.com/android/repository/build-tools_r29.0.3-linux.zip',
'5652d8cd5eaaade0b853bfe0ae6cbfa0706a6f70a0ebb25ca24a6f484ec3d855'),
2020-06-10 21:59:04 +02:00
('https://dl.google.com/android/repository/build-tools_r30-linux.zip',
'ed3b7f9b2d15e90a12c2e739adb749d7d834e2f953e677380206bd14db135c6c'),
2020-07-19 01:17:42 +02:00
('https://dl.google.com/android/repository/build-tools_r30.0.1-linux.zip',
'560eace2cc6ca16011fbb97c92c39aa0441d54dbfc13837dfbdb4a6bdf9c9da8'),
2020-09-16 14:58:50 +02:00
('https://dl.google.com/android/repository/build-tools_r30.0.2-linux.zip',
'565af786dc0cc1941002174fb945122eabd080b222cd4c7c3d9a2ae0fabf5dc4'),
2020-12-27 01:51:22 +01:00
('https://dl.google.com/android/repository/build-tools_r30.0.3-linux.zip',
'24593500aa95d2f99fb4f10658aae7e65cb519be6cd33fa164f15f27f3c4a2d6'),
2021-07-22 13:26:32 +02:00
('https://dl.google.com/android/repository/build-tools_r31-linux.zip',
'f90c22f5562638a2e00762e1711eebd55e7f0a05232b65200d387307d057bfe8'),
2021-03-01 22:34:31 +01:00
('https://services.gradle.org/distributions/gradle-6.8.3-bin.zip',
'7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205'),
2021-05-17 18:59:55 +02:00
('https://services.gradle.org/distributions/gradle-7.0.2-bin.zip',
'0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'),
2022-06-14 13:03:42 +02:00
('https://dl.google.com/android/repository/android-ndk-r23c-linux.zip',
'6ce94604b77d28113ecd588d425363624a5228d9662450c48d2e4053f8039242'),
2014-12-31 16:42:26 +01:00
]
2014-05-31 23:00:27 +02:00
2014-05-07 16:13:22 +02:00
2014-01-10 19:54:53 +01:00
def sha256_for_file(path):
2016-03-13 21:49:38 +01:00
with open(path, 'rb') as f:
2014-01-10 19:54:53 +01:00
s = hashlib.sha256()
while True:
data = f.read(4096)
if not data:
break
s.update(data)
return s.hexdigest()
2017-03-14 20:14:06 +01:00
2018-09-25 15:47:33 +02:00
def verify_file_sha256(path, sha256):
if sha256_for_file(path) != sha256:
2019-02-20 15:54:32 +01:00
logging.critical("File verification for '{path}' failed! "
"expected sha256 checksum: {checksum}"
.format(path=path, checksum=sha256))
2018-09-25 15:47:33 +02:00
sys.exit(1)
else:
2019-02-20 15:54:32 +01:00
logging.debug("sucessfully verifyed file '{path}' "
"('{checksum}')".format(path=path,
checksum=sha256))
2018-09-25 15:47:33 +02:00
def get_vagrant_home():
return os.environ.get('VAGRANT_HOME',
os.path.join(os.path.expanduser('~'),
'.vagrant.d'))
2016-06-23 16:38:39 +02:00
def run_via_vagrant_ssh(v, cmdlist):
if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
cmd = cmdlist
else:
cmd = ' '.join(cmdlist)
v._run_vagrant_command(['ssh', '-c', cmd])
2019-02-20 15:47:43 +01:00
def update_cache(cachedir):
2019-05-27 12:35:57 +02:00
count_files = 0
2019-02-20 15:47:43 +01:00
for srcurl, shasum in CACHE_FILES:
2016-09-25 18:55:29 +02:00
filename = os.path.basename(srcurl)
local_filename = os.path.join(cachedir, filename)
2019-05-27 12:35:57 +02:00
count_files = count_files + 1
2016-09-25 18:55:29 +02:00
if os.path.exists(local_filename):
2019-01-28 22:16:48 +01:00
if sha256_for_file(local_filename) == shasum:
2019-05-27 12:35:57 +02:00
logging.info("\t...shasum verified for '{filename}'\t({filecounter} of {filesum} files)".format(filename=local_filename, filecounter=count_files, filesum=len(CACHE_FILES)))
2019-01-28 22:16:48 +01:00
continue
2016-09-25 18:55:29 +02:00
local_length = os.path.getsize(local_filename)
else:
local_length = -1
resume_header = {}
download = True
try:
r = requests.head(srcurl, allow_redirects=True, timeout=60)
if r.status_code == 200:
content_length = int(r.headers.get('content-length'))
else:
content_length = local_length # skip the download
except requests.exceptions.RequestException as e:
content_length = local_length # skip the download
2019-02-20 15:54:32 +01:00
logging.warn('%s', e)
2016-09-25 18:55:29 +02:00
if local_length == content_length:
download = False
elif local_length > content_length:
2019-02-20 15:54:32 +01:00
logging.info('deleting corrupt file from cache: %s', local_filename)
2016-09-25 18:55:29 +02:00
os.remove(local_filename)
2019-02-20 15:54:32 +01:00
logging.info("Downloading %s to cache", filename)
2016-09-25 18:55:29 +02:00
elif local_length > -1 and local_length < content_length:
2019-02-20 15:54:32 +01:00
logging.info("Resuming download of %s", local_filename)
2016-09-25 18:55:29 +02:00
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
else:
2019-02-20 15:54:32 +01:00
logging.info("Downloading %s to cache", filename)
2016-06-10 11:42:42 +02:00
2016-09-25 18:55:29 +02:00
if download:
2019-05-28 12:07:36 +02:00
r = requests.get(srcurl, headers=resume_header, stream=True,
allow_redirects=True, timeout=60)
2016-06-10 11:42:42 +02:00
content_length = int(r.headers.get('content-length'))
2016-09-25 18:55:29 +02:00
with open(local_filename, 'ab') as f:
for chunk in progress.bar(r.iter_content(chunk_size=65536),
expected_size=(content_length / 65536) + 1):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
v = sha256_for_file(local_filename)
if v == shasum:
2019-05-27 12:35:57 +02:00
logging.info("\t...shasum verified for '{filename}'\t({filecounter} of {filesum} files)".format(filename=local_filename, filecounter=count_files, filesum=len(CACHE_FILES)))
2013-08-09 20:37:13 +02:00
else:
2019-02-20 15:54:32 +01:00
logging.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
2016-09-25 18:55:29 +02:00
os.remove(local_filename)
sys.exit(1)
2016-07-04 20:22:00 +02:00
2017-03-24 20:04:50 +01:00
2017-04-13 12:49:14 +02:00
def debug_log_vagrant_vm(vm_dir, config):
2017-03-24 20:04:50 +01:00
if options.verbosity >= 3:
_vagrant_dir = os.path.join(vm_dir, '.vagrant')
2019-02-20 15:54:32 +01:00
logging.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
logging.debug('> vagrant status')
2017-03-24 20:04:50 +01:00
subprocess.call(['vagrant', 'status'], cwd=vm_dir)
2019-02-20 15:54:32 +01:00
logging.debug('> vagrant box list')
2017-03-24 20:04:50 +01:00
subprocess.call(['vagrant', 'box', 'list'])
2017-04-13 12:49:14 +02:00
if config['vm_provider'] == 'libvirt':
2019-02-20 15:54:32 +01:00
logging.debug('> virsh -c qmeu:///system list --all')
2017-04-13 12:49:14 +02:00
subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
2017-05-23 11:59:53 +02:00
domain = 'buildserver_default'
2019-02-20 15:54:32 +01:00
logging.debug('> virsh -c qemu:///system snapshot-list %s', domain)
2017-05-23 11:59:53 +02:00
subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
2017-03-24 20:04:50 +01:00
def main():
2019-02-20 16:26:57 +01:00
global config, tail
2017-03-24 20:04:50 +01:00
if options.skip_cache_update:
2019-02-20 15:54:32 +01:00
logging.info('skipping cache update and verification...')
2017-03-24 20:04:50 +01:00
else:
2019-02-20 16:26:57 +01:00
update_cache(config['cachedir'])
2017-03-24 20:04:50 +01:00
2016-09-25 18:55:29 +02:00
# use VirtualBox software virtualization if hardware is not available,
# like if this is being run in kvm or some other VM platform, like
# http://jenkins.debian.net, the values are 'on' or 'off'
if sys.platform.startswith('darwin'):
# all < 10 year old Macs work, and OSX servers as VM host are very
# rare, but this could also be auto-detected if someone codes it
config['hwvirtex'] = 'on'
2019-02-20 15:54:32 +01:00
logging.info('platform is darwnin -> hwvirtex = \'on\'')
2016-09-25 18:55:29 +02:00
elif os.path.exists('/proc/cpuinfo'):
with open('/proc/cpuinfo') as f:
contents = f.read()
if 'vmx' in contents or 'svm' in contents:
config['hwvirtex'] = 'on'
2019-02-20 15:54:32 +01:00
logging.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
2016-09-25 18:55:29 +02:00
serverdir = os.path.join(os.getcwd(), 'buildserver')
logfilename = os.path.join(serverdir, 'up.log')
if not os.path.exists(logfilename):
open(logfilename, 'a').close() # create blank file
log_cm = vagrant.make_file_cm(logfilename)
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
2020-01-14 22:51:19 +01:00
# https://phoenhex.re/2018-03-25/not-a-vagrant-bug
os_env = os.environ.copy()
os_env['VAGRANT_DISABLE_VBOXSYMLINKCREATE'] = '1'
v.env = os_env
2016-09-25 18:55:29 +02:00
2017-03-24 20:04:50 +01:00
if options.verbosity >= 2:
2016-09-25 18:55:29 +02:00
tail = fdroidserver.tail.Tail(logfilename)
tail.start()
2017-03-26 03:15:33 +02:00
vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
2016-09-25 18:55:29 +02:00
if options.clean:
2017-03-26 01:41:39 +01:00
vm.destroy()
2016-09-25 18:55:29 +02:00
# Check against the existing Vagrantfile.yaml, and if they differ, we
# need to create a new box:
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
writevf = True
if os.path.exists(vf):
2019-02-20 15:54:32 +01:00
logging.info('Halting %s', serverdir)
2016-09-25 18:55:29 +02:00
v.halt()
with open(vf, 'r', encoding='utf-8') as f:
oldconfig = yaml.load(f)
if config != oldconfig:
2019-02-20 15:54:32 +01:00
logging.info("Server configuration has changed, rebuild from scratch is required")
2017-03-26 01:41:39 +01:00
vm.destroy()
2016-09-25 18:55:29 +02:00
else:
2019-02-20 15:54:32 +01:00
logging.info("Re-provisioning existing server")
2016-09-25 18:55:29 +02:00
writevf = False
else:
2019-02-20 15:54:32 +01:00
logging.info("No existing server - building from scratch")
2016-09-25 18:55:29 +02:00
if writevf:
with open(vf, 'w', encoding='utf-8') as f:
yaml.dump(config, f)
2016-06-20 11:33:46 +02:00
2018-07-28 16:48:15 +02:00
# Check if selected provider is supported
if config['vm_provider'] not in ['libvirt', 'virtualbox']:
2019-02-20 15:54:32 +01:00
logging.critical("Currently selected VM provider '{vm_provider}' "
"is not supported. (please choose from: "
"virtualbox, libvirt)"
.format(vm_provider=config['cm_provider']))
2018-07-28 16:48:15 +02:00
sys.exit(1)
2021-09-03 12:18:13 +02:00
# Check if selected Vagrant box is available
2018-07-28 16:48:15 +02:00
available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
2018-09-13 11:08:51 +02:00
if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
2019-02-20 15:54:32 +01:00
logging.critical("Vagrant box '{basebox}' not available "
"for '{vm_provider}' VM provider. "
"Please make sure it's added to vagrant. "
"(If you need a basebox to begin with, "
"here is how we're bootstrapping it: "
"https://gitlab.com/fdroid/basebox)"
.format(vm_provider=config['vm_provider'],
basebox=config['basebox']))
2018-07-28 16:48:15 +02:00
sys.exit(1)
2016-09-25 18:55:29 +02:00
2021-09-03 12:18:13 +02:00
# Download and verify pre-built Vagrant boxes
2018-09-25 15:47:33 +02:00
if config['basebox'] == BASEBOX_DEFAULT:
buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default'])
if buildserver_not_created or options.clean:
# make vagrant download and add basebox
target_basebox_installed = any([x for x in v.box_list() if x.name == BASEBOX_DEFAULT and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
if not target_basebox_installed:
cmd = [shutil.which('vagrant'), 'box', 'add', BASEBOX_DEFAULT,
'--box-version=' + config['basebox_version'],
'--provider=' + config['vm_provider']]
ret_val = subprocess.call(cmd)
if ret_val != 0:
2019-02-20 15:54:32 +01:00
logging.critical("downloading basebox '{box}' "
"({provider}, version {version}) failed."
.format(box=config['basebox'],
provider=config['vm_provider'],
version=config['basebox_version']))
2018-09-25 15:47:33 +02:00
sys.exit(1)
# verify box
if config['basebox_version'] not in BASEBOX_CHECKSUMS.keys():
2019-02-20 15:54:32 +01:00
logging.critical("can not verify '{box}', "
"unknown basebox version '{version}'"
.format(box=config['basebox'],
version=config['basebox_version']))
2018-09-25 15:47:33 +02:00
sys.exit(1)
for filename, sha256 in BASEBOX_CHECKSUMS[config['basebox_version']][config['vm_provider']].items():
verify_file_sha256(os.path.join(get_vagrant_home(),
'boxes',
BASEBOX_DEFAULT.replace('/', '-VAGRANTSLASH-'),
config['basebox_version'],
config['vm_provider'],
filename),
sha256)
2019-02-20 15:54:32 +01:00
logging.info("successfully verified: '{box}' "
"({provider}, version {version})"
.format(box=config['basebox'],
provider=config['vm_provider'],
version=config['basebox_version']))
2018-09-25 15:47:33 +02:00
else:
2019-02-20 15:54:32 +01:00
logging.debug('not updating basebox ...')
2018-09-25 15:47:33 +02:00
else:
2019-02-20 15:54:32 +01:00
logging.debug('using unverified basebox ...')
2018-09-25 15:47:33 +02:00
2019-02-20 15:54:32 +01:00
logging.info("Configuring build server VM")
2017-04-13 12:49:14 +02:00
debug_log_vagrant_vm(serverdir, config)
2017-03-25 01:56:20 +01:00
try:
2017-05-22 17:12:34 +02:00
v.up(provision=True)
2017-10-20 04:38:39 +02:00
except subprocess.CalledProcessError:
2017-04-13 12:49:14 +02:00
debug_log_vagrant_vm(serverdir, config)
2019-02-20 15:54:32 +01:00
logging.error("'vagrant up' failed.")
2017-03-25 01:56:20 +01:00
sys.exit(1)
2016-09-25 18:55:29 +02:00
2016-06-23 17:12:06 +02:00
if config['copy_caches_from_host']:
ssh_config = v.ssh_config()
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
fullpath = os.path.join(os.getenv('HOME'), d)
2021-03-12 13:14:55 +01:00
os.system('date')
print('rsyncing', fullpath, 'into VM')
2016-06-23 17:12:06 +02:00
if os.path.isdir(fullpath):
2017-12-07 11:52:10 +01:00
ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
'-o StrictHostKeyChecking=no',
'-o UserKnownHostsFile=/dev/null',
'-o LogLevel=FATAL',
'-o IdentitiesOnly=yes',
'-o PasswordAuthentication=no'))
2017-12-06 09:47:08 +01:00
# TODO vagrant 1.5+ provides `vagrant rsync`
2016-06-23 17:12:06 +02:00
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
2017-12-07 15:56:09 +01:00
subprocess.call(['rsync', '-ax', '--delete', '-e',
2017-12-06 09:47:08 +01:00
ssh_command,
2016-06-23 17:12:06 +02:00
fullpath + '/',
user + '@' + hostname + ':~/' + d + '/'])
# this file changes every time but should not be cached
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
2019-02-20 15:54:32 +01:00
logging.info("Stopping build server VM")
2016-09-27 08:49:32 +02:00
v.halt()
2013-05-31 08:48:39 +02:00
2019-02-20 15:54:32 +01:00
logging.info("Packaging")
2016-09-26 11:22:05 +02:00
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
if os.path.exists(boxfile):
os.remove(boxfile)
2017-02-14 22:58:35 +01:00
2017-05-22 15:50:32 +02:00
vm.package(output=boxfile)
2016-09-25 18:55:29 +02:00
2019-02-20 15:54:32 +01:00
logging.info("Adding box")
2017-05-22 17:14:48 +02:00
vm.box_add('buildserver', boxfile, force=True)
2016-09-25 18:55:29 +02:00
2017-05-22 17:14:48 +02:00
if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
2019-02-20 15:54:32 +01:00
logging.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
2017-05-22 17:12:34 +02:00
sys.exit(1)
2017-05-22 17:29:12 +02:00
if not options.keep_box_file:
2019-02-20 15:54:32 +01:00
logging.debug("""box added to vagrant, removing generated box file '%s'""",
boxfile)
2017-05-22 17:29:12 +02:00
os.remove(boxfile)
2016-09-25 18:55:29 +02:00
if __name__ == '__main__':
2018-10-23 16:34:32 +02:00
if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
logging.critical('This must be run as ./makebuildserver in fdroidserver.git!')
sys.exit(1)
2018-10-23 16:38:34 +02:00
if os.path.isfile('/usr/bin/systemd-detect-virt'):
try:
virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
except subprocess.CalledProcessError:
virt = 'none'
2022-09-05 17:14:51 +02:00
if virt in ('qemu', 'kvm', 'bochs'):
2018-10-23 16:38:34 +02:00
logging.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
config['vm_provider'] = 'libvirt'
elif virt != 'none':
logging.info('Running in an unsupported VM guest (%s)!', virt)
logging.debug('detected virt: %s', virt)
2016-09-25 18:55:29 +02:00
try:
main()
finally:
if tail is not None:
tail.stop()