mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-11 15:13:27 +02:00
692 lines
36 KiB
Python
Executable File
692 lines
36 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import re
|
|
import requests
|
|
import stat
|
|
import sys
|
|
import shutil
|
|
import subprocess
|
|
import vagrant
|
|
import hashlib
|
|
import yaml
|
|
import json
|
|
import logging
|
|
from clint.textui import progress
|
|
from optparse import OptionParser
|
|
import fdroidserver.tail
|
|
import fdroidserver.vmtools
|
|
|
|
|
|
parser = OptionParser()
|
|
parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
|
|
help="Spew out even more information than normal")
|
|
parser.add_option('-q', action='store_const', const=0, dest='verbosity')
|
|
parser.add_option("-c", "--clean", action="store_true", default=False,
|
|
help="Build from scratch, rather than attempting to update the existing server")
|
|
parser.add_option('--skip-cache-update', action="store_true", default=False,
|
|
help="""Skip downloading and checking cache."""
|
|
"""This assumes that the cache is already downloaded completely.""")
|
|
parser.add_option('--keep-box-file', action="store_true", default=False,
|
|
help="""Box file will not be deleted after adding it to box storage"""
|
|
""" (KVM-only).""")
|
|
options, args = parser.parse_args()
|
|
|
|
|
|
logformat = '%(levelname)s: %(message)s'
|
|
loglevel = logging.DEBUG
|
|
if options.verbosity == 1:
|
|
loglevel = logging.INFO
|
|
elif options.verbosity <= 0:
|
|
loglevel = logging.WARNING
|
|
logging.basicConfig(format=logformat, level=loglevel)
|
|
|
|
tail = None
|
|
|
|
BASEBOX_DEFAULT = 'fdroid/basebox-stretch64'
|
|
BASEBOX_VERSION_DEFAULT = '0.5.1'
|
|
BASEBOX_CHECKSUMS = {
|
|
'0.5.1': {
|
|
'libvirt': {
|
|
'box.img': 'ad015940b866e36a593ef5fa0035ec6703f74a7f082ab76a1d2bd9463714cd4a',
|
|
'metadata.json': '5ced8ecf886722a5152095e099b778b1d2b859c2e1dcf834182274034b8a629d',
|
|
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
|
|
},
|
|
'virtualbox': {
|
|
'box-disk1.vmdk': 'cba36a9c9814bdff9aabaea8786c27477ef8958cf6ee65ad844cb2726bdab93e',
|
|
'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
|
|
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
|
|
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
|
|
}
|
|
},
|
|
'0.5': {
|
|
'virtualbox': {
|
|
'box-disk1.vmdk': '8834d5eb78758437c2517f83282172fd5e3842d88f657d577592d0917cd02f89',
|
|
'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
|
|
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
|
|
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
|
|
},
|
|
'libvirt': {
|
|
'box.img': '2ef5f1fdc98c24a4f67cecb526d21e1d73dedf5a0072ceff528a0e75da3ff452',
|
|
'metadata.json': 'da79a5e2327dcf81a18a9d66a6e91205a20e440f23d3928e633fd39d60c641e5',
|
|
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
|
|
}
|
|
},
|
|
'0.3': {
|
|
'libvirt': {
|
|
'box.img': '24f06f415dde4cdb01d68c904fc57386ea060ba7b94e700670c58694b3d3635e',
|
|
'metadata.json': '0965955659082fd2e67723deb3311ba253c96153d3176d856db1b3e6e461cf23',
|
|
'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
|
|
},
|
|
'virtualbox': {
|
|
'box-disk1.vmdk': '103114977f1a36f7121ef9b3a1495129baa10bfedfada61a13345c8863c4dcd6',
|
|
'box.ovf': '33a5fbaf3dba443237baefcba6d56ca7a76121ca530f1140aa8263a69d7d3695',
|
|
'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
|
|
'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
|
|
}
|
|
}
|
|
}
|
|
|
|
config = {
|
|
'basebox': BASEBOX_DEFAULT,
|
|
'debian_mirror': 'http://deb.debian.org/debian/',
|
|
'apt_package_cache': False,
|
|
'copy_caches_from_host': False,
|
|
'boot_timeout': 600,
|
|
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
|
|
'cpus': 1,
|
|
'memory': 2048,
|
|
'hwvirtex': 'off',
|
|
'vm_provider': 'virtualbox',
|
|
}
|
|
|
|
# load config file, if present
|
|
if os.path.exists('makebuildserver.config.py'):
|
|
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
|
|
elif os.path.exists('makebs.config.py'):
|
|
# this is the old name for the config file
|
|
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
|
|
if '__builtins__' in config:
|
|
del(config['__builtins__']) # added by compile/exec
|
|
logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
|
|
if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config:
|
|
config['basebox_version'] = BASEBOX_VERSION_DEFAULT
|
|
# note: vagrant allows putting '/' into the name of a local box,
|
|
# so this check is not completely relyable, but better than nothing
|
|
if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']:
|
|
logging.critical("Can not get version '{version}' for basebox '{box}', "
|
|
"vagrant does not support versioning for locally added boxes."
|
|
.format(box=config['basebox'], version=config['basebox_version']))
|
|
|
|
# Update cached files.
|
|
if not os.path.exists(config['cachedir']):
|
|
os.makedirs(config['cachedir'], 0o755)
|
|
logging.debug('created cachedir {} because it did not exists.'.format(config['cachedir']))
|
|
|
|
if config['vm_provider'] == 'libvirt':
|
|
tmp = config['cachedir']
|
|
while tmp != '/':
|
|
mode = os.stat(tmp).st_mode
|
|
if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
|
|
logging.critical('ERROR: %s will not be accessible to the VM! To fix, run:', tmp)
|
|
logging.critical(' chmod a+X %s', tmp)
|
|
sys.exit(1)
|
|
tmp = os.path.dirname(tmp)
|
|
logging.debug('cache dir %s is accessible for libvirt vm.', config['cachedir'])
|
|
|
|
if config['apt_package_cache']:
|
|
config['aptcachedir'] = config['cachedir'] + '/apt/archives'
|
|
logging.debug('aptcachedir is set to %s', config['aptcachedir'])
|
|
aptcachelock = os.path.join(config['aptcachedir'], 'lock')
|
|
if os.path.isfile(aptcachelock):
|
|
logging.info('apt cache dir is locked, removing lock')
|
|
os.remove(aptcachelock)
|
|
aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
|
|
if os.path.isdir(aptcachepartial):
|
|
logging.info('removing partial downloads from apt cache dir')
|
|
shutil.rmtree(aptcachepartial)
|
|
|
|
CACHE_FILES = [
|
|
# Don't update sdk tools beyond 25.2.5.
|
|
# Support for android update project has been removed and there is no replacement.
|
|
# Until we find a solution for that we need to stay at this revision.
|
|
('https://dl.google.com/android/repository/tools_r25.2.5-linux.zip',
|
|
'577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e'),
|
|
('https://dl.google.com/android/repository/android_m2repository_r47.zip',
|
|
'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
|
|
('https://dl.google.com/android/repository/android-2.1_r03.zip',
|
|
'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
|
|
('https://dl.google.com/android/repository/android-2.2_r03.zip',
|
|
'7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
|
|
('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
|
|
'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
|
|
('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
|
|
'54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
|
|
('https://dl.google.com/android/repository/android-3.0_r02.zip',
|
|
'1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
|
|
('https://dl.google.com/android/repository/android-3.1_r03.zip',
|
|
'7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
|
|
('https://dl.google.com/android/repository/android-3.2_r01.zip',
|
|
'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
|
|
('https://dl.google.com/android/repository/android-14_r04.zip',
|
|
'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
|
|
('https://dl.google.com/android/repository/android-15_r05.zip',
|
|
'5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
|
|
('https://dl.google.com/android/repository/android-16_r05.zip',
|
|
'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
|
|
('https://dl.google.com/android/repository/android-17_r03.zip',
|
|
'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
|
|
('https://dl.google.com/android/repository/android-18_r03.zip',
|
|
'166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
|
|
('https://dl.google.com/android/repository/android-19_r04.zip',
|
|
'5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
|
|
('https://dl.google.com/android/repository/android-20_r02.zip',
|
|
'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
|
|
('https://dl.google.com/android/repository/android-21_r02.zip',
|
|
'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
|
|
('https://dl.google.com/android/repository/android-22_r02.zip',
|
|
'45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
|
|
('https://dl.google.com/android/repository/platform-23_r03.zip',
|
|
'4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
|
|
('https://dl.google.com/android/repository/platform-24_r02.zip',
|
|
'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
|
|
('https://dl.google.com/android/repository/platform-25_r03.zip',
|
|
'9b742d34590fe73fb7229e34835ecffb1846ca389d9f924f0b2a37de525dc6b8'),
|
|
('https://dl.google.com/android/repository/platform-26_r02.zip',
|
|
'2aafa7d19c5e9c4b643ee6ade3d85ef89dc2f79e8383efdb9baf7fddad74b52a'),
|
|
('https://dl.google.com/android/repository/platform-27_r03.zip',
|
|
'020c4c090bc82ce87ebaae5d1a922e21b39a1d03c78ffa43f0c3e42fc7d28169'),
|
|
('https://dl.google.com/android/repository/platform-28_r06.zip',
|
|
'8452dbbf9668a428abb243c4f02a943b7aa83af3cca627629a15c4c09f28e7bd'),
|
|
('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
|
|
'3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
|
|
('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
|
|
'296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
|
|
('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
|
|
'3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
|
|
('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
|
|
'91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
|
|
'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
|
|
'82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
|
|
'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
|
|
('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
|
|
'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
|
|
'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
|
|
'924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
|
|
'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
|
|
('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
|
|
'74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
|
|
'671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
|
|
'1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.3-linux.zip',
|
|
'152c1b187947edd10c65af8b279d40321ecc106106323e53df3608e578042d65'),
|
|
('https://dl.google.com/android/repository/build-tools_r26-linux.zip',
|
|
'7422682f92fb471d4aad4c053c9982a9a623377f9d5e4de7a73cd44ebf2f3c61'),
|
|
('https://dl.google.com/android/repository/build-tools_r26.0.1-linux.zip',
|
|
'c8617f25a7de2aeb9ddcacf1aeb413e053d5ed5ef4a3f31fe0ce21d4428ee0ea'),
|
|
('https://dl.google.com/android/repository/build-tools_r26.0.2-linux.zip',
|
|
'a752849fac85c4a7f9ea165ec8f367b0ebe8bbf6a1f33fc8605342be004231ce'),
|
|
('https://dl.google.com/android/repository/build-tools_r26.0.3-linux.zip',
|
|
'5c250c602b1657c4c70a6078925e9e01e5714526b707309bc1c708be6137a4db'),
|
|
('https://dl.google.com/android/repository/build-tools_r27-linux.zip',
|
|
'53d3322774a0bf229b372c0288108b4bfa27d74725fce8f0a3393e8df6b9ef22'),
|
|
('https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip',
|
|
'2e8e0946e93af50667ae02ef200e81c1ac2269b59f14955397245e9e441e8b1e'),
|
|
('https://dl.google.com/android/repository/build-tools_r27.0.2-linux.zip',
|
|
'e73674e065a93ffb05c30a15c8021c0d72ea7c3c206eb9020eb93e49e42ce851'),
|
|
('https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip',
|
|
'5e1f4fc5203f13de120c56f9cc103bb2e57d940959547506196ab10ddc9e6b97'),
|
|
('https://dl.google.com/android/repository/build-tools_r28-linux.zip',
|
|
'a7c6b73661836a6c50e32c06e7b8e8d6ef8c96f6812f269b9c90470dfb37753c'),
|
|
('https://dl.google.com/android/repository/build-tools_r28.0.1-linux.zip',
|
|
'4825b52a6258a9c2ab073720a5e91e2fa81b552b48024ee81887b863397416af'),
|
|
('https://dl.google.com/android/repository/build-tools_r28.0.2-linux.zip',
|
|
'12cebcafd8f30119c7ef53ffb3562a5b5b9f776c9399038587c18df44ea6452b'),
|
|
('https://dl.google.com/android/repository/build-tools_r28.0.3-linux.zip',
|
|
'7954956a40633c88f693d638cbc23f68e9e2499dc7a4b7dfdaf6a3e91387749a'),
|
|
('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
|
|
'420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
|
|
('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
|
|
'010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
|
|
('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
|
|
'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
|
|
('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
|
|
'3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
|
|
('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
|
|
'18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
|
|
('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
|
|
'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
|
|
('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
|
|
'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
|
|
('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
|
|
'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
|
|
('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
|
|
'66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
|
|
('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
|
|
'8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
|
|
('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
|
|
'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
|
|
('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
|
|
'0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
|
|
('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
|
|
'993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
|
|
('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
|
|
'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
|
|
('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
|
|
'39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
|
|
('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
|
|
'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
|
|
('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
|
|
'5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
|
|
('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
|
|
'9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
|
|
('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
|
|
'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
|
|
('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
|
|
'72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
|
|
('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
|
|
'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
|
|
('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
|
|
'0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
|
|
('https://services.gradle.org/distributions/gradle-3.5.1-bin.zip',
|
|
'8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8'),
|
|
('https://services.gradle.org/distributions/gradle-4.0-bin.zip',
|
|
'56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe'),
|
|
('https://services.gradle.org/distributions/gradle-4.0.1-bin.zip',
|
|
'd717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4'),
|
|
('https://services.gradle.org/distributions/gradle-4.0.2-bin.zip',
|
|
'79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22'),
|
|
('https://services.gradle.org/distributions/gradle-4.1-bin.zip',
|
|
'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43'),
|
|
('https://services.gradle.org/distributions/gradle-4.2-bin.zip',
|
|
'515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.2.1-bin.zip',
|
|
'b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.3-bin.zip',
|
|
'8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.3.1-bin.zip',
|
|
'15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.4-bin.zip',
|
|
'fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.4.1-bin.zip',
|
|
'e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.5-bin.zip',
|
|
'03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.5.1-bin.zip',
|
|
'3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.6-bin.zip',
|
|
'98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.7-bin.zip',
|
|
'fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.8-bin.zip',
|
|
'f3e29692a8faa94eb0b02ebf36fa263a642b3ae8694ef806c45c345b8683f1ba'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.8.1-bin.zip',
|
|
'af334d994b5e69e439ab55b5d2b7d086da5ea6763d78054f49f147b06370ed71'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.9-bin.zip',
|
|
'e66e69dce8173dd2004b39ba93586a184628bc6c28461bc771d6835f7f9b0d28'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.10-bin.zip',
|
|
'248cfd92104ce12c5431ddb8309cf713fe58de8e330c63176543320022f59f18'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.10.1-bin.zip',
|
|
'e53ce3a01cf016b5d294eef20977ad4e3c13e761ac1e475f1ffad4c6141a92bd'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.10.2-bin.zip',
|
|
'b49c6da1b2cb67a0caf6c7480630b51c70a11ca2016ff2f555eaeda863143a29'),
|
|
('https://downloads.gradle.org/distributions/gradle-4.10.3-bin.zip',
|
|
'8626cbf206b4e201ade7b87779090690447054bc93f052954c78480fa6ed186e'),
|
|
('https://downloads.gradle.org/distributions/gradle-5.0-bin.zip',
|
|
'6157ac9f3410bc63644625b3b3e9e96c963afd7910ae0697792db57813ee79a6'),
|
|
('https://downloads.gradle.org/distributions/gradle-5.1.1-bin.zip',
|
|
'4953323605c5d7b89e97d0dc7779e275bccedefcdac090aec123375eae0cc798'),
|
|
('https://downloads.gradle.org/distributions/gradle-5.2.1-bin.zip',
|
|
'748c33ff8d216736723be4037085b8dc342c6a0f309081acf682c9803e407357'),
|
|
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
|
|
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
|
|
('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
|
|
'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
|
|
('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
|
|
'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
|
|
('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
|
|
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
|
|
('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
|
|
'0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
|
|
('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
|
|
'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
|
|
('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip',
|
|
'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'),
|
|
('https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip',
|
|
'5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd'),
|
|
('https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip',
|
|
'4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd'),
|
|
('https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip',
|
|
'0fbb1645d0f1de4dde90a4ff79ca5ec4899c835e729d692f433fda501623257a'),
|
|
]
|
|
|
|
|
|
def sha256_for_file(path):
|
|
with open(path, 'rb') as f:
|
|
s = hashlib.sha256()
|
|
while True:
|
|
data = f.read(4096)
|
|
if not data:
|
|
break
|
|
s.update(data)
|
|
return s.hexdigest()
|
|
|
|
|
|
def verify_file_sha256(path, sha256):
|
|
if sha256_for_file(path) != sha256:
|
|
logging.critical("File verification for '{path}' failed! "
|
|
"expected sha256 checksum: {checksum}"
|
|
.format(path=path, checksum=sha256))
|
|
sys.exit(1)
|
|
else:
|
|
logging.debug("sucessfully verifyed file '{path}' "
|
|
"('{checksum}')".format(path=path,
|
|
checksum=sha256))
|
|
|
|
|
|
def get_vagrant_home():
|
|
return os.environ.get('VAGRANT_HOME',
|
|
os.path.join(os.path.expanduser('~'),
|
|
'.vagrant.d'))
|
|
|
|
|
|
def run_via_vagrant_ssh(v, cmdlist):
|
|
if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
|
|
cmd = cmdlist
|
|
else:
|
|
cmd = ' '.join(cmdlist)
|
|
v._run_vagrant_command(['ssh', '-c', cmd])
|
|
|
|
|
|
def update_cache(cachedir):
|
|
for srcurl, shasum in CACHE_FILES:
|
|
filename = os.path.basename(srcurl)
|
|
local_filename = os.path.join(cachedir, filename)
|
|
|
|
if os.path.exists(local_filename):
|
|
if sha256_for_file(local_filename) == shasum:
|
|
logging.info("\t...shasum verified for %s", local_filename)
|
|
continue
|
|
local_length = os.path.getsize(local_filename)
|
|
else:
|
|
local_length = -1
|
|
|
|
resume_header = {}
|
|
download = True
|
|
|
|
try:
|
|
r = requests.head(srcurl, allow_redirects=True, timeout=60)
|
|
if r.status_code == 200:
|
|
content_length = int(r.headers.get('content-length'))
|
|
else:
|
|
content_length = local_length # skip the download
|
|
except requests.exceptions.RequestException as e:
|
|
content_length = local_length # skip the download
|
|
logging.warn('%s', e)
|
|
|
|
if local_length == content_length:
|
|
download = False
|
|
elif local_length > content_length:
|
|
logging.info('deleting corrupt file from cache: %s', local_filename)
|
|
os.remove(local_filename)
|
|
logging.info("Downloading %s to cache", filename)
|
|
elif local_length > -1 and local_length < content_length:
|
|
logging.info("Resuming download of %s", local_filename)
|
|
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
|
|
else:
|
|
logging.info("Downloading %s to cache", filename)
|
|
|
|
if download:
|
|
r = requests.get(srcurl, headers=resume_header,
|
|
stream=True, allow_redirects=True)
|
|
content_length = int(r.headers.get('content-length'))
|
|
with open(local_filename, 'ab') as f:
|
|
for chunk in progress.bar(r.iter_content(chunk_size=65536),
|
|
expected_size=(content_length / 65536) + 1):
|
|
if chunk: # filter out keep-alive new chunks
|
|
f.write(chunk)
|
|
|
|
v = sha256_for_file(local_filename)
|
|
if v == shasum:
|
|
logging.info("\t...shasum verified for %s", local_filename)
|
|
else:
|
|
logging.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
|
|
os.remove(local_filename)
|
|
sys.exit(1)
|
|
|
|
|
|
def debug_log_vagrant_vm(vm_dir, config):
|
|
if options.verbosity >= 3:
|
|
_vagrant_dir = os.path.join(vm_dir, '.vagrant')
|
|
logging.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
|
|
logging.debug('> vagrant status')
|
|
subprocess.call(['vagrant', 'status'], cwd=vm_dir)
|
|
logging.debug('> vagrant box list')
|
|
subprocess.call(['vagrant', 'box', 'list'])
|
|
if config['vm_provider'] == 'libvirt':
|
|
logging.debug('> virsh -c qmeu:///system list --all')
|
|
subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
|
|
domain = 'buildserver_default'
|
|
logging.debug('> virsh -c qemu:///system snapshot-list %s', domain)
|
|
subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
|
|
|
|
|
|
def main():
|
|
global config, tail
|
|
|
|
if options.skip_cache_update:
|
|
logging.info('skipping cache update and verification...')
|
|
else:
|
|
update_cache(config['cachedir'])
|
|
|
|
# use VirtualBox software virtualization if hardware is not available,
|
|
# like if this is being run in kvm or some other VM platform, like
|
|
# http://jenkins.debian.net, the values are 'on' or 'off'
|
|
if sys.platform.startswith('darwin'):
|
|
# all < 10 year old Macs work, and OSX servers as VM host are very
|
|
# rare, but this could also be auto-detected if someone codes it
|
|
config['hwvirtex'] = 'on'
|
|
logging.info('platform is darwnin -> hwvirtex = \'on\'')
|
|
elif os.path.exists('/proc/cpuinfo'):
|
|
with open('/proc/cpuinfo') as f:
|
|
contents = f.read()
|
|
if 'vmx' in contents or 'svm' in contents:
|
|
config['hwvirtex'] = 'on'
|
|
logging.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
|
|
|
|
serverdir = os.path.join(os.getcwd(), 'buildserver')
|
|
logfilename = os.path.join(serverdir, 'up.log')
|
|
if not os.path.exists(logfilename):
|
|
open(logfilename, 'a').close() # create blank file
|
|
log_cm = vagrant.make_file_cm(logfilename)
|
|
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
|
|
|
|
if options.verbosity >= 2:
|
|
tail = fdroidserver.tail.Tail(logfilename)
|
|
tail.start()
|
|
|
|
vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
|
|
if options.clean:
|
|
vm.destroy()
|
|
|
|
# Check against the existing Vagrantfile.yaml, and if they differ, we
|
|
# need to create a new box:
|
|
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
|
writevf = True
|
|
if os.path.exists(vf):
|
|
logging.info('Halting %s', serverdir)
|
|
v.halt()
|
|
with open(vf, 'r', encoding='utf-8') as f:
|
|
oldconfig = yaml.load(f)
|
|
if config != oldconfig:
|
|
logging.info("Server configuration has changed, rebuild from scratch is required")
|
|
vm.destroy()
|
|
else:
|
|
logging.info("Re-provisioning existing server")
|
|
writevf = False
|
|
else:
|
|
logging.info("No existing server - building from scratch")
|
|
if writevf:
|
|
with open(vf, 'w', encoding='utf-8') as f:
|
|
yaml.dump(config, f)
|
|
|
|
# Check if selected provider is supported
|
|
if config['vm_provider'] not in ['libvirt', 'virtualbox']:
|
|
logging.critical("Currently selected VM provider '{vm_provider}' "
|
|
"is not supported. (please choose from: "
|
|
"virtualbox, libvirt)"
|
|
.format(vm_provider=config['cm_provider']))
|
|
sys.exit(1)
|
|
# Check if selected basebox is available
|
|
available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
|
|
if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
|
|
logging.critical("Vagrant box '{basebox}' not available "
|
|
"for '{vm_provider}' VM provider. "
|
|
"Please make sure it's added to vagrant. "
|
|
"(If you need a basebox to begin with, "
|
|
"here is how we're bootstrapping it: "
|
|
"https://gitlab.com/fdroid/basebox)"
|
|
.format(vm_provider=config['vm_provider'],
|
|
basebox=config['basebox']))
|
|
sys.exit(1)
|
|
|
|
# download and verfiy fdroid pre-built basebox
|
|
if config['basebox'] == BASEBOX_DEFAULT:
|
|
buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default'])
|
|
if buildserver_not_created or options.clean:
|
|
# make vagrant download and add basebox
|
|
target_basebox_installed = any([x for x in v.box_list() if x.name == BASEBOX_DEFAULT and x.provider == config['vm_provider'] and x.version == config['basebox_version']])
|
|
if not target_basebox_installed:
|
|
cmd = [shutil.which('vagrant'), 'box', 'add', BASEBOX_DEFAULT,
|
|
'--box-version=' + config['basebox_version'],
|
|
'--provider=' + config['vm_provider']]
|
|
ret_val = subprocess.call(cmd)
|
|
if ret_val != 0:
|
|
logging.critical("downloading basebox '{box}' "
|
|
"({provider}, version {version}) failed."
|
|
.format(box=config['basebox'],
|
|
provider=config['vm_provider'],
|
|
version=config['basebox_version']))
|
|
sys.exit(1)
|
|
# verify box
|
|
if config['basebox_version'] not in BASEBOX_CHECKSUMS.keys():
|
|
logging.critical("can not verify '{box}', "
|
|
"unknown basebox version '{version}'"
|
|
.format(box=config['basebox'],
|
|
version=config['basebox_version']))
|
|
sys.exit(1)
|
|
for filename, sha256 in BASEBOX_CHECKSUMS[config['basebox_version']][config['vm_provider']].items():
|
|
verify_file_sha256(os.path.join(get_vagrant_home(),
|
|
'boxes',
|
|
BASEBOX_DEFAULT.replace('/', '-VAGRANTSLASH-'),
|
|
config['basebox_version'],
|
|
config['vm_provider'],
|
|
filename),
|
|
sha256)
|
|
logging.info("successfully verified: '{box}' "
|
|
"({provider}, version {version})"
|
|
.format(box=config['basebox'],
|
|
provider=config['vm_provider'],
|
|
version=config['basebox_version']))
|
|
else:
|
|
logging.debug('not updating basebox ...')
|
|
else:
|
|
logging.debug('using unverified basebox ...')
|
|
|
|
logging.info("Configuring build server VM")
|
|
debug_log_vagrant_vm(serverdir, config)
|
|
try:
|
|
v.up(provision=True)
|
|
except subprocess.CalledProcessError:
|
|
debug_log_vagrant_vm(serverdir, config)
|
|
logging.error("'vagrant up' failed.")
|
|
sys.exit(1)
|
|
|
|
if config['copy_caches_from_host']:
|
|
ssh_config = v.ssh_config()
|
|
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
|
|
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
|
|
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
|
|
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
|
|
|
|
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
|
|
fullpath = os.path.join(os.getenv('HOME'), d)
|
|
if os.path.isdir(fullpath):
|
|
ssh_command = ' '.join(('ssh -i {0} -p {1}'.format(key, port),
|
|
'-o StrictHostKeyChecking=no',
|
|
'-o UserKnownHostsFile=/dev/null',
|
|
'-o LogLevel=FATAL',
|
|
'-o IdentitiesOnly=yes',
|
|
'-o PasswordAuthentication=no'))
|
|
# TODO vagrant 1.5+ provides `vagrant rsync`
|
|
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
|
|
subprocess.call(['rsync', '-ax', '--delete', '-e',
|
|
ssh_command,
|
|
fullpath + '/',
|
|
user + '@' + hostname + ':~/' + d + '/'])
|
|
|
|
# this file changes every time but should not be cached
|
|
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
|
|
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
|
|
|
|
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
buildserverid = p.communicate()[0].strip()
|
|
logging.info("Writing buildserver ID ...ID is %s", buildserverid)
|
|
run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
|
|
|
|
logging.info("Stopping build server VM")
|
|
v.halt()
|
|
|
|
logging.info("Packaging")
|
|
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
|
|
if os.path.exists(boxfile):
|
|
os.remove(boxfile)
|
|
|
|
vm.package(output=boxfile)
|
|
|
|
logging.info("Adding box")
|
|
vm.box_add('buildserver', boxfile, force=True)
|
|
|
|
if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
|
|
logging.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
|
|
sys.exit(1)
|
|
|
|
if not options.keep_box_file:
|
|
logging.debug("""box added to vagrant, removing generated box file '%s'""",
|
|
boxfile)
|
|
os.remove(boxfile)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
|
|
logging.critical('This must be run as ./makebuildserver in fdroidserver.git!')
|
|
sys.exit(1)
|
|
|
|
if os.path.isfile('/usr/bin/systemd-detect-virt'):
|
|
try:
|
|
virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
|
|
except subprocess.CalledProcessError:
|
|
virt = 'none'
|
|
if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
|
|
logging.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
|
|
config['vm_provider'] = 'libvirt'
|
|
elif virt != 'none':
|
|
logging.info('Running in an unsupported VM guest (%s)!', virt)
|
|
logging.debug('detected virt: %s', virt)
|
|
|
|
try:
|
|
main()
|
|
finally:
|
|
if tail is not None:
|
|
tail.stop()
|