mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-04 22:40:12 +01:00
7a1d236c8d
Since I discovered there is an r10e zipball, this can now get all NDKs in zipball form. fdroid/android-sdk-transparency-log@447fea86e7 closes #902
631 lines
32 KiB
Python
Executable File
631 lines
32 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import re
|
|
import requests
|
|
import stat
|
|
import sys
|
|
import shutil
|
|
import subprocess
|
|
import vagrant
|
|
import hashlib
|
|
import yaml
|
|
import json
|
|
import logging
|
|
from clint.textui import progress
|
|
from optparse import OptionParser
|
|
import fdroidserver.tail
|
|
import fdroidserver.vmtools
|
|
|
|
|
|
parser = OptionParser()
|
|
parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
|
|
help="Spew out even more information than normal")
|
|
parser.add_option('-q', action='store_const', const=0, dest='verbosity')
|
|
parser.add_option("-c", "--clean", action="store_true", default=False,
|
|
help="Build from scratch, rather than attempting to update the existing server")
|
|
parser.add_option('--skip-cache-update', action="store_true", default=False,
|
|
help="""Skip downloading and checking cache."""
|
|
"""This assumes that the cache is already downloaded completely.""")
|
|
parser.add_option('--keep-box-file', action="store_true", default=False,
|
|
help="""Box file will not be deleted after adding it to box storage"""
|
|
""" (KVM-only).""")
|
|
options, args = parser.parse_args()
|
|
|
|
|
|
logformat = '%(levelname)s: %(message)s'
|
|
loglevel = logging.DEBUG
|
|
if options.verbosity == 1:
|
|
loglevel = logging.INFO
|
|
elif options.verbosity <= 0:
|
|
loglevel = logging.WARNING
|
|
logging.basicConfig(format=logformat, level=loglevel)
|
|
|
|
tail = None
|
|
|
|
BASEBOX_DEFAULT = 'fdroid/basebox-stretch64'
|
|
BASEBOX_VERSION_DEFAULT = '0.5.1'
|
|
BASEBOX_CHECKSUMS = {
|
|
'0.6.0': {
|
|
'libvirt': {
|
|
'box.img': '82c2c3548cf48f0f4c6601f40f8bec36ff37e9a74d6f717067a526250ad790ad',
|
|
'metadata.json': '9b5f62362ce3cd25c50881d8ae124879fc21ed4fdb16cc78d57058f116680f25',
|
|
'Vagrantfile': '4435901624f21dad201c3bd7f0d8d4ece842bc9fbbb70e312eee54f07173f24e',
|
|
},
|
|
'virtualbox': {
|
|
'box-disk1.vmdk': '6b536f26dcee137aca9a3f5f6f20aef795193ef2e8c387a0ffbdb7c5fe2ec0fb',
|
|
'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
|
|
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
|
|
'Vagrantfile': '95c64a0e82a6420845c05038c4c97b3aba629b09eb2b78e879423d06f6b54a54',
|
|
}
|
|
},
|
|
'0.5.1': {
|
|
'libvirt': {
|
|
'box.img': 'ad015940b866e36a593ef5fa0035ec6703f74a7f082ab76a1d2bd9463714cd4a',
|
|
'metadata.json': '5ced8ecf886722a5152095e099b778b1d2b859c2e1dcf834182274034b8a629d',
|
|
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
|
|
},
|
|
'virtualbox': {
|
|
'box-disk1.vmdk': 'cba36a9c9814bdff9aabaea8786c27477ef8958cf6ee65ad844cb2726bdab93e',
|
|
'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
|
|
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
|
|
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
|
|
}
|
|
},
|
|
'0.5': {
|
|
'virtualbox': {
|
|
'box-disk1.vmdk': '8834d5eb78758437c2517f83282172fd5e3842d88f657d577592d0917cd02f89',
|
|
'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
|
|
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
|
|
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
|
|
},
|
|
'libvirt': {
|
|
'box.img': '2ef5f1fdc98c24a4f67cecb526d21e1d73dedf5a0072ceff528a0e75da3ff452',
|
|
'metadata.json': 'da79a5e2327dcf81a18a9d66a6e91205a20e440f23d3928e633fd39d60c641e5',
|
|
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
|
|
}
|
|
},
|
|
'0.3': {
|
|
'libvirt': {
|
|
'box.img': '24f06f415dde4cdb01d68c904fc57386ea060ba7b94e700670c58694b3d3635e',
|
|
'metadata.json': '0965955659082fd2e67723deb3311ba253c96153d3176d856db1b3e6e461cf23',
|
|
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
|
|
},
|
|
'virtualbox': {
|
|
'box-disk1.vmdk': '103114977f1a36f7121ef9b3a1495129baa10bfedfada61a13345c8863c4dcd6',
|
|
'box.ovf': '33a5fbaf3dba443237baefcba6d56ca7a76121ca530f1140aa8263a69d7d3695',
|
|
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
|
|
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
|
|
}
|
|
}
|
|
}
|
|
|
|
config = {
|
|
'basebox': BASEBOX_DEFAULT,
|
|
'debian_mirror': 'http://deb.debian.org/debian/',
|
|
'apt_package_cache': False,
|
|
'copy_caches_from_host': False,
|
|
'boot_timeout': 600,
|
|
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
|
|
'cpus': 1,
|
|
'memory': 2048,
|
|
'hwvirtex': 'off',
|
|
'vm_provider': 'virtualbox',
|
|
}
|
|
|
|
# load config file, if present
|
|
if os.path.exists('makebuildserver.config.py'):
|
|
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
|
|
elif os.path.exists('makebs.config.py'):
|
|
# this is the old name for the config file
|
|
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
|
|
if '__builtins__' in config:
|
|
del(config['__builtins__']) # added by compile/exec
|
|
logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
|
|
if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config:
|
|
config['basebox_version'] = BASEBOX_VERSION_DEFAULT
|
|
# note: vagrant allows putting '/' into the name of a local box,
|
|
# so this check is not completely relyable, but better than nothing
|
|
if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']:
|
|
logging.critical("Can not get version '{version}' for basebox '{box}', "
|
|
"vagrant does not support versioning for locally added boxes."
|
|
.format(box=config['basebox'], version=config['basebox_version']))
|
|
|
|
# Update cached files.
|
|
if not os.path.exists(config['cachedir']):
|
|
os.makedirs(config['cachedir'], 0o755)
|
|
logging.debug('created cachedir {} because it did not exists.'.format(config['cachedir']))
|
|
|
|
if config['vm_provider'] == 'libvirt':
|
|
tmp = config['cachedir']
|
|
while tmp != '/':
|
|
mode = os.stat(tmp).st_mode
|
|
if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
|
|
logging.critical('ERROR: %s will not be accessible to the VM! To fix, run:', tmp)
|
|
logging.critical(' chmod a+X %s', tmp)
|
|
sys.exit(1)
|
|
tmp = os.path.dirname(tmp)
|
|
logging.debug('cache dir %s is accessible for libvirt vm.', config['cachedir'])
|
|
|
|
if config['apt_package_cache']:
|
|
config['aptcachedir'] = config['cachedir'] + '/apt/archives'
|
|
logging.debug('aptcachedir is set to %s', config['aptcachedir'])
|
|
aptcachelock = os.path.join(config['aptcachedir'], 'lock')
|
|
if os.path.isfile(aptcachelock):
|
|
logging.info('apt cache dir is locked, removing lock')
|
|
os.remove(aptcachelock)
|
|
aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
|
|
if os.path.isdir(aptcachepartial):
|
|
logging.info('removing partial downloads from apt cache dir')
|
|
shutil.rmtree(aptcachepartial)
|
|
|
|
CACHE_FILES = [
|
|
# Don't update sdk tools beyond 25.2.5.
|
|
# Support for android update project has been removed and there is no replacement.
|
|
# Until we find a solution for that we need to stay at this revision.
|
|
('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
|
|
'577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
|
|
('https://dl.google.com/android/repository/android_m2repository_r47.zip',
|
|
'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
|
|
('https://dl.google.com/android/repository/android-2.1_r03.zip',
|
|
'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
|
|
('https://dl.google.com/android/repository/android-2.2_r03.zip',
|
|
'7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
|
|
('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
|
|
'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
|
|
('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
|
|
'54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
|
|
('https://dl.google.com/android/repository/android-3.0_r02.zip',
|
|
'1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
|
|
('https://dl.google.com/android/repository/android-3.1_r03.zip',
|
|
'7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
|
|
('https://dl.google.com/android/repository/android-3.2_r01.zip',
|
|
'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
|
|
('https://dl.google.com/android/repository/android-14_r04.zip',
|
|
'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
|
|
('https://dl.google.com/android/repository/android-15_r05.zip',
|
|
'5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
|
|
('https://dl.google.com/android/repository/android-16_r05.zip',
|
|
'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
|
|
('https://dl.google.com/android/repository/android-17_r03.zip',
|
|
'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
|
|
('https://dl.google.com/android/repository/android-18_r03.zip',
|
|
'166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
|
|
('https://dl.google.com/android/repository/android-19_r04.zip',
|
|
'5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
|
|
('https://dl.google.com/android/repository/android-20_r02.zip',
|
|
'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
|
|
('https://dl.google.com/android/repository/android-21_r02.zip',
|
|
'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
|
|
('https://dl.google.com/android/repository/android-22_r02.zip',
|
|
'45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
|
|
('https://dl.google.com/android/repository/platform-23_r03.zip',
|
|
'4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
|
|
('https://dl.google.com/android/repository/platform-24_r02.zip',
|
|
'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
|
|
('https://dl.google.com/android/repository/platform-25_r03.zip',
|
|
'9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
|
|
('https://dl.google.com/android/repository/platform-26_r02.zip',
|
|
'2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
|
|
('https://dl.google.com/android/repository/platform-27_r03.zip',
|
|
'020c4c090bc82ce87ebaae5d1a922e21b39a1d03c78ffa43f0c3e42fc7d28169'),
|
|
('https://dl.google.com/android/repository/platform-28_r06.zip',
|
|
'8452dbbf9668a428abb243c4f02a943b7aa83af3cca627629a15c4c09f28e7bd'),
|
|
('https://dl.google.com/android/repository/platform-29_r05.zip',
|
|
'951da8bf175254da74626824f919bd28def64f8828f29dd3b124a535cf4049d8'),
|
|
('https://dl.google.com/android/repository/platform-30_r03.zip',
|
|
'f3f5b75744dbf6ee6ed3e8174a71e513bfee502d0bc3463ea97e517bff68d84e'),
|
|
('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
|
|
'3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
|
|
('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
|
|
'296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
|
|
('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
|
|
'3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
|
|
('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
|
|
'91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
|
|
'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
|
|
'82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
|
|
'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
|
|
('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
|
|
'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
|
|
'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
|
|
'924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
|
|
'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
|
|
('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
|
|
'74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
|
|
'671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
|
|
'1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
|
|
'152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
|
|
('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
|
|
'7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
|
|
('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
|
|
'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
|
|
('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
|
|
'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
|
|
('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
|
|
'5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
|
|
('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
|
|
'53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
|
|
('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
|
|
'2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
|
|
('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
|
|
'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
|
|
('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
|
|
'5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
|
|
('https://dl.google.com/android/repository/build-tools_r28-linux.zip',
|
|
'a7c6b73661836a6c50e32c06e7b8e8d6ef8c96f6812f269b9c90470dfb37753c'),
|
|
('https://dl.google.com/android/repository/build-tools_r28.0.1-linux.zip',
|
|
'4825b52a6258a9c2ab073720a5e91e2fa81b552b48024ee81887b863397416af'),
|
|
('https://dl.google.com/android/repository/build-tools_r28.0.2-linux.zip',
|
|
'12cebcafd8f30119c7ef53ffb3562a5b5b9f776c9399038587c18df44ea6452b'),
|
|
('https://dl.google.com/android/repository/build-tools_r28.0.3-linux.zip',
|
|
'7954956a40633c88f693d638cbc23f68e9e2499dc7a4b7dfdaf6a3e91387749a'),
|
|
('https://dl.google.com/android/repository/build-tools_r29.0.2-linux.zip',
|
|
'1e9393cbfd4a4b82e30e7f55ab38db4a5a3259db93d5821c63597bc74522fa08'),
|
|
('https://dl.google.com/android/repository/build-tools_r29.0.3-linux.zip',
|
|
'5652d8cd5eaaade0b853bfe0ae6cbfa0706a6f70a0ebb25ca24a6f484ec3d855'),
|
|
('https://dl.google.com/android/repository/build-tools_r30-linux.zip',
|
|
'ed3b7f9b2d15e90a12c2e739adb749d7d834e2f953e677380206bd14db135c6c'),
|
|
('https://dl.google.com/android/repository/build-tools_r30.0.1-linux.zip',
|
|
'560eace2cc6ca16011fbb97c92c39aa0441d54dbfc13837dfbdb4a6bdf9c9da8'),
|
|
('https://dl.google.com/android/repository/build-tools_r30.0.2-linux.zip',
|
|
'565af786dc0cc1941002174fb945122eabd080b222cd4c7c3d9a2ae0fabf5dc4'),
|
|
('https://dl.google.com/android/repository/build-tools_r30.0.3-linux.zip',
|
|
'24593500aa95d2f99fb4f10658aae7e65cb519be6cd33fa164f15f27f3c4a2d6'),
|
|
('https://services.gradle.org/distributions/gradle-6.8.2-bin.zip',
|
|
'8de6efc274ab52332a9c820366dd5cf5fc9d35ec7078fd70c8ec6913431ee610'),
|
|
('https://services.gradle.org/distributions/gradle-6.8.3-bin.zip',
|
|
'7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205'),
|
|
('https://services.gradle.org/distributions/gradle-7.0-bin.zip',
|
|
'eb8b89184261025b0430f5b2233701ff1377f96da1ef5e278af6ae8bac5cc305'),
|
|
('https://services.gradle.org/distributions/gradle-7.0.1-bin.zip',
|
|
'dccda8aa069563c8ba2f6cdfd0777df0e34a5b4d15138ca8b9757e94f4e8a8cb'),
|
|
('https://services.gradle.org/distributions/gradle-7.0.2-bin.zip',
|
|
'0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'),
|
|
('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
|
|
'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'),
|
|
('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip',
|
|
'ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b'),
|
|
]
|
|
|
|
|
|
def sha256_for_file(path):
|
|
with open(path, 'rb') as f:
|
|
s = hashlib.sha256()
|
|
while True:
|
|
data = f.read(4096)
|
|
if not data:
|
|
break
|
|
s.update(data)
|
|
return s.hexdigest()
|
|
|
|
|
|
def verify_file_sha256(path, sha256):
|
|
if sha256_for_file(path) != sha256:
|
|
logging.critical("File verification for '{path}' failed! "
|
|
"expected sha256 checksum: {checksum}"
|
|
.format(path=path, checksum=sha256))
|
|
sys.exit(1)
|
|
else:
|
|
logging.debug("sucessfully verifyed file '{path}' "
|
|
"('{checksum}')".format(path=path,
|
|
checksum=sha256))
|
|
|
|
|
|
def get_vagrant_home():
|
|
return os.environ.get('VAGRANT_HOME',
|
|
os.path.join(os.path.expanduser('~'),
|
|
'.vagrant.d'))
|
|
|
|
|
|
def run_via_vagrant_ssh(v, cmdlist):
|
|
if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
|
|
cmd = cmdlist
|
|
else:
|
|
cmd = ' '.join(cmdlist)
|
|
v._run_vagrant_command(['ssh', '-c', cmd])
|
|
|
|
|
|
def update_cache(cachedir):
|
|
count_files = 0
|
|
for srcurl, shasum in CACHE_FILES:
|
|
filename = os.path.basename(srcurl)
|
|
local_filename = os.path.join(cachedir, filename)
|
|
count_files = count_files + 1
|
|
if os.path.exists(local_filename):
|
|
if sha256_for_file(local_filename) == shasum:
|
|
logging.info("\t...shasum verified for '{filename}'\t({filecounter} of {filesum} files)".format(filename=local_filename, filecounter=count_files, filesum=len(CACHE_FILES)))
|
|
continue
|
|
local_length = os.path.getsize(local_filename)
|
|
else:
|
|
local_length = -1
|
|
|
|
resume_header = {}
|
|
download = True
|
|
|
|
try:
|
|
r = requests.head(srcurl, allow_redirects=True, timeout=60)
|
|
if r.status_code == 200:
|
|
content_length = int(r.headers.get('content-length'))
|
|
else:
|
|
content_length = local_length # skip the download
|
|
except requests.exceptions.RequestException as e:
|
|
content_length = local_length # skip the download
|
|
logging.warn('%s', e)
|
|
|
|
if local_length == content_length:
|
|
download = False
|
|
elif local_length > content_length:
|
|
logging.info('deleting corrupt file from cache: %s', local_filename)
|
|
os.remove(local_filename)
|
|
logging.info("Downloading %s to cache", filename)
|
|
elif local_length > -1 and local_length < content_length:
|
|
logging.info("Resuming download of %s", local_filename)
|
|
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
|
|
else:
|
|
logging.info("Downloading %s to cache", filename)
|
|
|
|
if download:
|
|
r = requests.get(srcurl, headers=resume_header, stream=True,
|
|
allow_redirects=True, timeout=60)
|
|
content_length = int(r.headers.get('content-length'))
|
|
with open(local_filename, 'ab') as f:
|
|
for chunk in progress.bar(r.iter_content(chunk_size=65536),
|
|
expected_size=(content_length / 65536) + 1):
|
|
if chunk: # filter out keep-alive new chunks
|
|
f.write(chunk)
|
|
|
|
v = sha256_for_file(local_filename)
|
|
if v == shasum:
|
|
logging.info("\t...shasum verified for '{filename}'\t({filecounter} of {filesum} files)".format(filename=local_filename, filecounter=count_files, filesum=len(CACHE_FILES)))
|
|
else:
|
|
logging.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
|
|
os.remove(local_filename)
|
|
sys.exit(1)
|
|
|
|
|
|
def debug_log_vagrant_vm(vm_dir, config):
|
|
if options.verbosity >= 3:
|
|
_vagrant_dir = os.path.join(vm_dir, '.vagrant')
|
|
logging.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
|
|
logging.debug('> vagrant status')
|
|
subprocess.call(['vagrant', 'status'], cwd=vm_dir)
|
|
logging.debug('> vagrant box list')
|
|
subprocess.call(['vagrant', 'box', 'list'])
|
|
if config['vm_provider'] == 'libvirt':
|
|
logging.debug('> virsh -c qmeu:///system list --all')
|
|
subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
|
|
domain = 'buildserver_default'
|
|
logging.debug('> virsh -c qemu:///system snapshot-list %s', domain)
|
|
subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
|
|
|
|
|
|
def main():
|
|
global config, tail
|
|
|
|
if options.skip_cache_update:
|
|
logging.info('skipping cache update and verification...')
|
|
else:
|
|
update_cache(config['cachedir'])
|
|
|
|
# use VirtualBox software virtualization if hardware is not available,
|
|
# like if this is being run in kvm or some other VM platform, like
|
|
# http://jenkins.debian.net, the values are 'on' or 'off'
|
|
if sys.platform.startswith('darwin'):
|
|
# all < 10 year old Macs work, and OSX servers as VM host are very
|
|
# rare, but this could also be auto-detected if someone codes it
|
|
config['hwvirtex'] = 'on'
|
|
logging.info('platform is darwnin -> hwvirtex = \'on\'')
|
|
elif os.path.exists('/proc/cpuinfo'):
|
|
with open('/proc/cpuinfo') as f:
|
|
contents = f.read()
|
|
if 'vmx' in contents or 'svm' in contents:
|
|
config['hwvirtex'] = 'on'
|
|
logging.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
|
|
|
|
serverdir = os.path.join(os.getcwd(), 'buildserver')
|
|
logfilename = os.path.join(serverdir, 'up.log')
|
|
if not os.path.exists(logfilename):
|
|
open(logfilename, 'a').close() # create blank file
|
|
log_cm = vagrant.make_file_cm(logfilename)
|
|
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
|
|
# https://phoenhex.re/2018-03-25/not-a-vagrant-bug
|
|
os_env = os.environ.copy()
|
|
os_env['VAGRANT_DISABLE_VBOXSYMLINKCREATE'] = '1'
|
|
v.env = os_env
|
|
|
|
if options.verbosity >= 2:
|
|
tail = fdroidserver.tail.Tail(logfilename)
|
|
tail.start()
|
|
|
|
vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
|
|
if options.clean:
|
|
vm.destroy()
|
|
|
|
# Check against the existing Vagrantfile.yaml, and if they differ, we
|
|
# need to create a new box:
|
|
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
|
writevf = True
|
|
if os.path.exists(vf):
|
|
logging.info('Halting %s', serverdir)
|
|
v.halt()
|
|
with open(vf, 'r', encoding='utf-8') as f:
|
|
oldconfig = yaml.load(f)
|
|
if config != oldconfig:
|
|
logging.info("Server configuration has changed, rebuild from scratch is required")
|
|
vm.destroy()
|
|
else:
|
|
logging.info("Re-provisioning existing server")
|
|
writevf = False
|
|
else:
|
|
logging.info("No existing server - building from scratch")
|
|
if writevf:
|
|
with open(vf, 'w', encoding='utf-8') as f:
|
|
yaml.dump(config, f)
|
|
|
|
# Check if selected provider is supported
|
|
if config['vm_provider'] not in ['libvirt', 'virtualbox']:
|
|
logging.critical("Currently selected VM provider '{vm_provider}' "
|
|
"is not supported. (please choose from: "
|
|
"virtualbox, libvirt)"
|
|
.format(vm_provider=config['cm_provider']))
|
|
sys.exit(1)
|
|
# Check if selected basebox is available
|
|
available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
|
|
if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
|
|
logging.critical("Vagrant box '{basebox}' not available "
|
|
"for '{vm_provider}' VM provider. "
|
|
"Please make sure it's added to vagrant. "
|
|
"(If you need a basebox to begin with, "
|
|
"here is how we're bootstrapping it: "
|
|
"https://gitlab.com/fdroid/basebox)"
|
|
.format(vm_provider=config['vm_provider'],
|
|
basebox=config['basebox']))
|
|
sys.exit(1)
|
|
|
|
# download and verfiy fdroid pre-built basebox
|
|
if config['basebox'] == BASEBOX_DEFAULT:
|
|
buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default'])
|
|
if buildserver_not_created or options.clean:
|
|
# make vagrant download and add basebox
|
|
target_basebox_installed = any([x for x in v.box_list() if x.name == BASEBOX_DEFAULT and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
|
|
if not target_basebox_installed:
|
|
cmd = [shutil.which('vagrant'), 'box', 'add', BASEBOX_DEFAULT,
|
|
'--box-version=' + config['basebox_version'],
|
|
'--provider=' + config['vm_provider']]
|
|
ret_val = subprocess.call(cmd)
|
|
if ret_val != 0:
|
|
logging.critical("downloading basebox '{box}' "
|
|
"({provider}, version {version}) failed."
|
|
.format(box=config['basebox'],
|
|
provider=config['vm_provider'],
|
|
version=config['basebox_version']))
|
|
sys.exit(1)
|
|
# verify box
|
|
if config['basebox_version'] not in BASEBOX_CHECKSUMS.keys():
|
|
logging.critical("can not verify '{box}', "
|
|
"unknown basebox version '{version}'"
|
|
.format(box=config['basebox'],
|
|
version=config['basebox_version']))
|
|
sys.exit(1)
|
|
for filename, sha256 in BASEBOX_CHECKSUMS[config['basebox_version']][config['vm_provider']].items():
|
|
verify_file_sha256(os.path.join(get_vagrant_home(),
|
|
'boxes',
|
|
BASEBOX_DEFAULT.replace('/', '-VAGRANTSLASH-'),
|
|
config['basebox_version'],
|
|
config['vm_provider'],
|
|
filename),
|
|
sha256)
|
|
logging.info("successfully verified: '{box}' "
|
|
"({provider}, version {version})"
|
|
.format(box=config['basebox'],
|
|
provider=config['vm_provider'],
|
|
version=config['basebox_version']))
|
|
else:
|
|
logging.debug('not updating basebox ...')
|
|
else:
|
|
logging.debug('using unverified basebox ...')
|
|
|
|
logging.info("Configuring build server VM")
|
|
debug_log_vagrant_vm(serverdir, config)
|
|
try:
|
|
v.up(provision=True)
|
|
except subprocess.CalledProcessError:
|
|
debug_log_vagrant_vm(serverdir, config)
|
|
logging.error("'vagrant up' failed.")
|
|
sys.exit(1)
|
|
|
|
if config['copy_caches_from_host']:
|
|
ssh_config = v.ssh_config()
|
|
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
|
|
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
|
|
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
|
|
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
|
|
|
|
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
|
|
fullpath = os.path.join(os.getenv('HOME'), d)
|
|
os.system('date')
|
|
print('rsyncing', fullpath, 'into VM')
|
|
if os.path.isdir(fullpath):
|
|
ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
|
|
'-o StrictHostKeyChecking=no',
|
|
'-o UserKnownHostsFile=/dev/null',
|
|
'-o LogLevel=FATAL',
|
|
'-o IdentitiesOnly=yes',
|
|
'-o PasswordAuthentication=no'))
|
|
# TODO vagrant 1.5+ provides `vagrant rsync`
|
|
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
|
|
subprocess.call(['rsync', '-ax', '--delete', '-e',
|
|
ssh_command,
|
|
fullpath + '/',
|
|
user + '@' + hostname + ':~/' + d + '/'])
|
|
|
|
# this file changes every time but should not be cached
|
|
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
|
|
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
|
|
|
|
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
buildserverid = p.communicate()[0].strip()
|
|
logging.info("Writing buildserver ID ...ID is %s", buildserverid)
|
|
# sync data before we halt() the machine, we had an empty buildserverid otherwise
|
|
write_bsid_cmd = 'sh -c "echo \'{}\' >/home/vagrant/buildserverid; sync"'.format(buildserverid)
|
|
run_via_vagrant_ssh(v, write_bsid_cmd)
|
|
logging.debug("+ {}".format(write_bsid_cmd))
|
|
|
|
logging.info("Stopping build server VM")
|
|
v.halt()
|
|
|
|
logging.info("Packaging")
|
|
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
|
|
if os.path.exists(boxfile):
|
|
os.remove(boxfile)
|
|
|
|
vm.package(output=boxfile)
|
|
|
|
logging.info("Adding box")
|
|
vm.box_add('buildserver', boxfile, force=True)
|
|
|
|
if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
|
|
logging.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
|
|
sys.exit(1)
|
|
|
|
if not options.keep_box_file:
|
|
logging.debug("""box added to vagrant, removing generated box file '%s'""",
|
|
boxfile)
|
|
os.remove(boxfile)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
|
|
logging.critical('This must be run as ./makebuildserver in fdroidserver.git!')
|
|
sys.exit(1)
|
|
|
|
if os.path.isfile('/usr/bin/systemd-detect-virt'):
|
|
try:
|
|
virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
|
|
except subprocess.CalledProcessError:
|
|
virt = 'none'
|
|
if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
|
|
logging.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
|
|
config['vm_provider'] = 'libvirt'
|
|
elif virt != 'none':
|
|
logging.info('Running in an unsupported VM guest (%s)!', virt)
|
|
logging.debug('detected virt: %s', virt)
|
|
|
|
try:
|
|
main()
|
|
finally:
|
|
if tail is not None:
|
|
tail.stop()
|