mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
988ac21e7f
This prevents v.destroy() from running if Vagrantfile.yaml does not exist, since that is required for vagrant to run: is the core config including the name of the box, etc. Otherwise, it would exit with an error. This also does complete cleanup when using libvirt.
510 lines
25 KiB
Python
Executable File
510 lines
25 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import requests
|
|
import stat
|
|
import sys
|
|
import subprocess
|
|
import vagrant
|
|
import hashlib
|
|
import yaml
|
|
from clint.textui import progress
|
|
from optparse import OptionParser
|
|
import fdroidserver.tail
|
|
|
|
|
|
if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
|
|
print('This must be run as ./makebuildserver in fdroidserver.git!')
|
|
sys.exit(1)
|
|
|
|
|
|
tail = None
|
|
|
|
parser = OptionParser()
|
|
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
|
help="Spew out even more information than normal")
|
|
parser.add_option("-c", "--clean", action="store_true", default=False,
|
|
help="Build from scratch, rather than attempting to update the existing server")
|
|
options, args = parser.parse_args()
|
|
|
|
# set up default config
|
|
cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
|
|
config = {
|
|
'basebox': 'jessie64',
|
|
'baseboxurl': [
|
|
pathlib.Path(os.path.join(cachedir, 'jessie64.box')).as_uri(),
|
|
'https://f-droid.org/jessie64.box',
|
|
],
|
|
'debian_mirror': 'http://http.debian.net/debian/',
|
|
'apt_package_cache': False,
|
|
'copy_caches_from_host': False,
|
|
'boot_timeout': 600,
|
|
'cachedir': cachedir,
|
|
'cpus': 1,
|
|
'memory': 1024,
|
|
'hwvirtex': 'off',
|
|
'vm_provider': 'virtualbox',
|
|
}
|
|
|
|
if os.path.isfile('/usr/bin/systemd-detect-virt'):
|
|
try:
|
|
virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
|
|
except subprocess.CalledProcessError as e:
|
|
virt = 'none'
|
|
if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
|
|
print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
|
|
config['vm_provider'] = 'libvirt'
|
|
elif virt != 'none':
|
|
print('Running in an unsupported VM guest (' + virt + ')!')
|
|
|
|
# load config file, if present
|
|
if os.path.exists('makebuildserver.config.py'):
|
|
exec(compile(open('makebuildserver.config.py').read(), 'makebuildserver.config.py', 'exec'), config)
|
|
elif os.path.exists('makebs.config.py'):
|
|
# this is the old name for the config file
|
|
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
|
|
if '__builtins__' in config:
|
|
del(config['__builtins__']) # added by compile/exec
|
|
|
|
# Update cached files.
|
|
cachedir = config['cachedir']
|
|
if not os.path.exists(cachedir):
|
|
os.makedirs(cachedir, 0o755)
|
|
|
|
if config['vm_provider'] == 'libvirt':
|
|
tmp = cachedir
|
|
while tmp != '/':
|
|
mode = os.stat(tmp).st_mode
|
|
if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
|
|
print('ERROR:', tmp, 'will not be accessible to the VM! To fix, run:')
|
|
print(' chmod a+X', tmp)
|
|
sys.exit(1)
|
|
tmp = os.path.dirname(tmp)
|
|
|
|
if config['apt_package_cache']:
|
|
config['aptcachedir'] = cachedir + '/apt/archives'
|
|
|
|
cachefiles = [
|
|
('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
|
|
'1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560'),
|
|
('https://dl.google.com/android/repository/android_m2repository_r47.zip',
|
|
'a3f91808dce50c1717737de90c18479ed3a78b147e06985247d138e7ab5123d0'),
|
|
('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
|
|
'85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
|
|
('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
|
|
'a8c4e3b32269c6b04c2adeabd112fce42f292dab1a40ef3b08ea7d4212be0df4'),
|
|
('https://dl.google.com/android/repository/android-2.0_r01-linux.zip',
|
|
'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
|
|
('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
|
|
'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
|
|
('https://dl.google.com/android/repository/android-2.1_r03.zip',
|
|
'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
|
|
('https://dl.google.com/android/repository/android-2.2_r03.zip',
|
|
'7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
|
|
('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
|
|
'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
|
|
('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
|
|
'54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
|
|
('https://dl.google.com/android/repository/android-3.0_r02.zip',
|
|
'1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
|
|
('https://dl.google.com/android/repository/android-3.1_r03.zip',
|
|
'7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
|
|
('https://dl.google.com/android/repository/android-3.2_r01.zip',
|
|
'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
|
|
('https://dl.google.com/android/repository/android-14_r04.zip',
|
|
'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
|
|
('https://dl.google.com/android/repository/android-15_r05.zip',
|
|
'5bc1f93aae86b4336ffc4cae9eb8ec41a9a8fd677582dd86a9629798f019bed9'),
|
|
('https://dl.google.com/android/repository/android-16_r05.zip',
|
|
'fd7f269a423d1f1d079eabf9f918ceab49108702a1c6bb2589d57c23393503d3'),
|
|
('https://dl.google.com/android/repository/android-17_r03.zip',
|
|
'b66e73fb2639f8c916fde4369aa29012a5c531e156dbb205fe3788fe998fbbe8'),
|
|
('https://dl.google.com/android/repository/android-18_r03.zip',
|
|
'166ae9cf299747a5faa8f04168f0ee47cd7466a975d8b44acaaa62a43e767568'),
|
|
('https://dl.google.com/android/repository/android-19_r04.zip',
|
|
'5efc3a3a682c1d49128daddb6716c433edf16e63349f32959b6207524ac04039'),
|
|
('https://dl.google.com/android/repository/android-20_r02.zip',
|
|
'ef08c453e16ab6e656cf5d9413ef61cb8c650607d33b24ee4ce08dafdfe965a7'),
|
|
('https://dl.google.com/android/repository/android-21_r02.zip',
|
|
'a76cd7ad3080ac6ce9f037cb935b399a1bad396c0605d4ff42f693695f1dcefe'),
|
|
('https://dl.google.com/android/repository/android-22_r02.zip',
|
|
'45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
|
|
('https://dl.google.com/android/repository/platform-23_r03.zip',
|
|
'4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
|
|
('https://dl.google.com/android/repository/platform-24_r02.zip',
|
|
'f268f5945c6ece7ea95c1c252067280854d2a20da924e22ae4720287df8bdbc9'),
|
|
('https://dl.google.com/android/repository/platform-25_r01.zip',
|
|
'da519dc3e07b8cb879265c94f798262c1f90791dfaa8b745d34883891378438e'),
|
|
('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
|
|
'4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
|
|
('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
|
|
'a9b7b1bdfd864780fdd03fa1683f3fe712a4276cf200646833808cb9159bafc0'),
|
|
('https://dl.google.com/android/repository/build-tools_r18.1-linux.zip',
|
|
'0753606738f31cc346426db1d46b7d021bc1bdaff63085f9ee9d278ee054d3c9'),
|
|
('https://dl.google.com/android/repository/build-tools_r18.1.1-linux.zip',
|
|
'7e4ed326b53078f4f23276ddab52c400011f7593dfbb6508c0a6671954dba8b0'),
|
|
('https://dl.google.com/android/repository/build-tools_r19-linux.zip',
|
|
'9442e1c5212ed594e344a231fa93e7a017a5ef8cc661117011f1d3142eca7acc'),
|
|
('https://dl.google.com/android/repository/build-tools_r19.0.1-linux.zip',
|
|
'b068edaff05c3253a63e9c8f0e1786429799b7e4b01514a847a8b291beb9232e'),
|
|
('https://dl.google.com/android/repository/build-tools_r19.0.2-linux.zip',
|
|
'06124fad0d4bde21191240d61df2059a8546c085064a9a57d024c36fa2c9bebb'),
|
|
('https://dl.google.com/android/repository/build-tools_r19.0.3-linux.zip',
|
|
'bc9b3db0de4a3e233a170274293359051a758f1e3f0d0d852ff4ad6d90d0a794'),
|
|
('https://dl.google.com/android/repository/build-tools_r19.1-linux.zip',
|
|
'3833b409f78c002a83244e220be380ea6fa44d604e0d47de4b7e5daefe7cd3f4'),
|
|
('https://dl.google.com/android/repository/build-tools_r20-linux.zip',
|
|
'296e09d62095d80e6eaa06a64cfa4c6f9f317c2d67ad8da6514523ec66f5c871'),
|
|
('https://dl.google.com/android/repository/build-tools_r21-linux.zip',
|
|
'12b818f38fe1b68091b94545988317438efbf41eb61fd36b72cd79f536044065'),
|
|
('https://dl.google.com/android/repository/build-tools_r21.0.1-linux.zip',
|
|
'a8922e80d3dd0cf6df14b29a7862448fa111b48086c639168d4b18c92431f559'),
|
|
('https://dl.google.com/android/repository/build-tools_r21.0.2-linux.zip',
|
|
'859b17a6b65d063dfd86c163489b736b12bdeecd9173fdddb3e9f32e0fe584b7'),
|
|
('https://dl.google.com/android/repository/build-tools_r21.1-linux.zip',
|
|
'022a85b92360272379b2f04b8a4d727e754dbe7eb8ab5a9568190e33e480d8f1'),
|
|
('https://dl.google.com/android/repository/build-tools_r21.1.1-linux.zip',
|
|
'29b612484de6b5cde0df6de655e413f7611b0557b440538397afa69b557e2f08'),
|
|
('https://dl.google.com/android/repository/build-tools_r21.1.2-linux.zip',
|
|
'3f88efc2d5316fb73f547f35b472610eed5e6f3f56762750ddad1c7d1d81660d'),
|
|
('https://dl.google.com/android/repository/build-tools_r22-linux.zip',
|
|
'061c021243f04c80c19568a6e3a027c00d8e269c9311d7bf07fced60fbde7bd5'),
|
|
('https://dl.google.com/android/repository/build-tools_r22.0.1-linux.zip',
|
|
'91e5524bf227aad1135ddd10905518ac49f74797d33d48920dcf8364b9fde214'),
|
|
('https://dl.google.com/android/repository/build-tools_r23-linux.zip',
|
|
'56bf4fc6c43638c55fef4a0937bad38281945725459841879b436c6922df786c'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.1-linux.zip',
|
|
'e56b3ef7b760ad06a7cee9b2d52ba7f43133dcecedfa5357f8845b3a80aeeecf'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.2-linux.zip',
|
|
'82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
|
|
('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
|
|
'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
|
|
('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
|
|
'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.1-linux.zip',
|
|
'a38ac637db357a31e33e38248399cb0edcc15040dca041370da38b6daf50c84d'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.2-linux.zip',
|
|
'924e29b8a189afbd119d44eae450fc0c9f197ed6f835df223931e45007987d95'),
|
|
('https://dl.google.com/android/repository/build-tools_r24.0.3-linux.zip',
|
|
'f2c02eb1d7e41ce314b5dac50440e7595380c4dd45b41ea1d7b0f86e49516927'),
|
|
('https://dl.google.com/android/repository/build-tools_r25-linux.zip',
|
|
'74eb6931fd7a56859bd8e35d8d73ca8fe7ba6bfd4b7ffe560fe58b7354f2e3aa'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.1-linux.zip',
|
|
'671b4e00f5b986c7355507c7024b725a4b4cadf11ca61fa5b1334ec6ea57d94f'),
|
|
('https://dl.google.com/android/repository/build-tools_r25.0.2-linux.zip',
|
|
'1d7ac9b6def16fb0254ec23c135c02dd9f6908073352a20315a017e4b2a904b0'),
|
|
# the binaries that Google uses are here:
|
|
# https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
|
|
('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
|
|
'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
|
|
('https://services.gradle.org/distributions/gradle-1.6-bin.zip',
|
|
'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
|
|
('https://services.gradle.org/distributions/gradle-1.7-bin.zip',
|
|
'360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
|
|
('https://services.gradle.org/distributions/gradle-1.8-bin.zip',
|
|
'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
|
|
('https://services.gradle.org/distributions/gradle-1.9-bin.zip',
|
|
'097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
|
|
('https://services.gradle.org/distributions/gradle-1.10-bin.zip',
|
|
'6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
|
|
('https://services.gradle.org/distributions/gradle-1.11-bin.zip',
|
|
'07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
|
|
('https://services.gradle.org/distributions/gradle-1.12-bin.zip',
|
|
'8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
|
|
('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
|
|
'3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
|
|
('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
|
|
'91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
|
|
('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
|
|
'420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
|
|
('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
|
|
'010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
|
|
('https://services.gradle.org/distributions/gradle-2.4-bin.zip',
|
|
'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
|
|
('https://services.gradle.org/distributions/gradle-2.5-bin.zip',
|
|
'3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
|
|
('https://services.gradle.org/distributions/gradle-2.6-bin.zip',
|
|
'18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
|
|
('https://services.gradle.org/distributions/gradle-2.7-bin.zip',
|
|
'cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06'),
|
|
('https://services.gradle.org/distributions/gradle-2.8-bin.zip',
|
|
'a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb'),
|
|
('https://services.gradle.org/distributions/gradle-2.9-bin.zip',
|
|
'c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f'),
|
|
('https://services.gradle.org/distributions/gradle-2.10-bin.zip',
|
|
'66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13'),
|
|
('https://services.gradle.org/distributions/gradle-2.11-bin.zip',
|
|
'8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6'),
|
|
('https://services.gradle.org/distributions/gradle-2.12-bin.zip',
|
|
'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
|
|
('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
|
|
'0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
|
|
('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
|
|
'993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
|
|
('https://services.gradle.org/distributions/gradle-2.14.1-bin.zip',
|
|
'cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39'),
|
|
('https://services.gradle.org/distributions/gradle-3.0-bin.zip',
|
|
'39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80'),
|
|
('https://services.gradle.org/distributions/gradle-3.1-bin.zip',
|
|
'c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc'),
|
|
('https://services.gradle.org/distributions/gradle-3.2-bin.zip',
|
|
'5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935'),
|
|
('https://services.gradle.org/distributions/gradle-3.2.1-bin.zip',
|
|
'9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689'),
|
|
('https://services.gradle.org/distributions/gradle-3.3-bin.zip',
|
|
'c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292'),
|
|
('https://services.gradle.org/distributions/gradle-3.4-bin.zip',
|
|
'72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205'),
|
|
('https://services.gradle.org/distributions/gradle-3.4.1-bin.zip',
|
|
'db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2'),
|
|
('https://services.gradle.org/distributions/gradle-3.5-bin.zip',
|
|
'0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110'),
|
|
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
|
|
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
|
|
('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
|
|
'8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
|
|
('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
|
|
'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
|
|
('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
|
|
'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
|
|
('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
|
|
'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
|
|
('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
|
|
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
|
|
('https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip',
|
|
'3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44'),
|
|
('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
|
|
'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
|
|
]
|
|
|
|
|
|
def sha256_for_file(path):
|
|
with open(path, 'rb') as f:
|
|
s = hashlib.sha256()
|
|
while True:
|
|
data = f.read(4096)
|
|
if not data:
|
|
break
|
|
s.update(data)
|
|
return s.hexdigest()
|
|
|
|
|
|
def destroy_current_image(v, serverdir):
|
|
global config
|
|
|
|
# cannot run vagrant without the config in the YAML file
|
|
if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')):
|
|
v.destroy()
|
|
elif options.verbose:
|
|
print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!')
|
|
if config['vm_provider'] == 'libvirt':
|
|
import libvirt
|
|
try:
|
|
domain = 'buildserver_default'
|
|
virConnect = libvirt.open('qemu:///system')
|
|
virDomain = virConnect.lookupByName(domain)
|
|
if virDomain:
|
|
virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE
|
|
| libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA
|
|
| libvirt.VIR_DOMAIN_UNDEFINE_NVRAM)
|
|
storagePool = virConnect.storagePoolLookupByName('default')
|
|
if storagePool:
|
|
for vol in storagePool.listAllVolumes():
|
|
vol.delete()
|
|
except libvirt.libvirtError as e:
|
|
print(e)
|
|
|
|
|
|
def run_via_vagrant_ssh(v, cmdlist):
|
|
if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
|
|
cmd = cmdlist
|
|
else:
|
|
cmd = ' '.join(cmdlist)
|
|
v._run_vagrant_command(['ssh', '-c', cmd])
|
|
|
|
|
|
def main():
|
|
global cachedir, cachefiles, config, tail
|
|
|
|
for srcurl, shasum in cachefiles:
|
|
filename = os.path.basename(srcurl)
|
|
local_filename = os.path.join(cachedir, filename)
|
|
|
|
if os.path.exists(local_filename):
|
|
local_length = os.path.getsize(local_filename)
|
|
else:
|
|
local_length = -1
|
|
|
|
resume_header = {}
|
|
download = True
|
|
|
|
try:
|
|
r = requests.head(srcurl, allow_redirects=True, timeout=60)
|
|
if r.status_code == 200:
|
|
content_length = int(r.headers.get('content-length'))
|
|
else:
|
|
content_length = local_length # skip the download
|
|
except requests.exceptions.RequestException as e:
|
|
content_length = local_length # skip the download
|
|
print(e)
|
|
|
|
if local_length == content_length:
|
|
download = False
|
|
elif local_length > content_length:
|
|
print('deleting corrupt file from cache: ' + local_filename)
|
|
os.remove(local_filename)
|
|
print("Downloading " + filename + " to cache")
|
|
elif local_length > -1 and local_length < content_length:
|
|
print("Resuming download of " + local_filename)
|
|
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
|
|
else:
|
|
print("Downloading " + filename + " to cache")
|
|
|
|
if download:
|
|
r = requests.get(srcurl, headers=resume_header,
|
|
stream=True, verify=False, allow_redirects=True)
|
|
content_length = int(r.headers.get('content-length'))
|
|
with open(local_filename, 'ab') as f:
|
|
for chunk in progress.bar(r.iter_content(chunk_size=65536),
|
|
expected_size=(content_length / 65536) + 1):
|
|
if chunk: # filter out keep-alive new chunks
|
|
f.write(chunk)
|
|
|
|
v = sha256_for_file(local_filename)
|
|
if v == shasum:
|
|
print("\t...shasum verified for " + local_filename)
|
|
else:
|
|
print("Invalid shasum of '" + v + "' detected for " + local_filename)
|
|
os.remove(local_filename)
|
|
sys.exit(1)
|
|
|
|
local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
|
|
print("Setting executable bit for " + local_qt_filename)
|
|
os.chmod(local_qt_filename, 0o755)
|
|
|
|
# use VirtualBox software virtualization if hardware is not available,
|
|
# like if this is being run in kvm or some other VM platform, like
|
|
# http://jenkins.debian.net, the values are 'on' or 'off'
|
|
if sys.platform.startswith('darwin'):
|
|
# all < 10 year old Macs work, and OSX servers as VM host are very
|
|
# rare, but this could also be auto-detected if someone codes it
|
|
config['hwvirtex'] = 'on'
|
|
elif os.path.exists('/proc/cpuinfo'):
|
|
with open('/proc/cpuinfo') as f:
|
|
contents = f.read()
|
|
if 'vmx' in contents or 'svm' in contents:
|
|
config['hwvirtex'] = 'on'
|
|
|
|
serverdir = os.path.join(os.getcwd(), 'buildserver')
|
|
logfilename = os.path.join(serverdir, 'up.log')
|
|
if not os.path.exists(logfilename):
|
|
open(logfilename, 'a').close() # create blank file
|
|
log_cm = vagrant.make_file_cm(logfilename)
|
|
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
|
|
|
|
if options.verbose:
|
|
tail = fdroidserver.tail.Tail(logfilename)
|
|
tail.start()
|
|
|
|
if options.clean:
|
|
destroy_current_image(v, serverdir)
|
|
|
|
# Check against the existing Vagrantfile.yaml, and if they differ, we
|
|
# need to create a new box:
|
|
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
|
writevf = True
|
|
if os.path.exists(vf):
|
|
print('Halting', serverdir)
|
|
v.halt()
|
|
with open(vf, 'r', encoding='utf-8') as f:
|
|
oldconfig = yaml.load(f)
|
|
if config != oldconfig:
|
|
print("Server configuration has changed, rebuild from scratch is required")
|
|
destroy_current_image(v, serverdir)
|
|
else:
|
|
print("Re-provisioning existing server")
|
|
writevf = False
|
|
else:
|
|
print("No existing server - building from scratch")
|
|
if writevf:
|
|
with open(vf, 'w', encoding='utf-8') as f:
|
|
yaml.dump(config, f)
|
|
|
|
if config['vm_provider'] == 'libvirt':
|
|
found_basebox = False
|
|
needs_mutate = False
|
|
for box in v.box_list():
|
|
if box.name == config['basebox']:
|
|
found_basebox = True
|
|
if box.provider != 'libvirt':
|
|
needs_mutate = True
|
|
continue
|
|
if not found_basebox:
|
|
if isinstance(config['baseboxurl'], str):
|
|
baseboxurl = config['baseboxurl']
|
|
else:
|
|
baseboxurl = config['baseboxurl'][0]
|
|
print('Adding', config['basebox'], 'from', baseboxurl)
|
|
v.box_add(config['basebox'], baseboxurl)
|
|
needs_mutate = True
|
|
if needs_mutate:
|
|
print('Converting', config['basebox'], 'to libvirt format')
|
|
v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
|
|
print('Removing virtualbox format copy of', config['basebox'])
|
|
v.box_remove(config['basebox'], 'virtualbox')
|
|
|
|
print("Configuring build server VM")
|
|
v.up(provision=True)
|
|
|
|
if config['copy_caches_from_host']:
|
|
ssh_config = v.ssh_config()
|
|
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
|
|
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
|
|
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
|
|
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
|
|
|
|
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
|
|
fullpath = os.path.join(os.getenv('HOME'), d)
|
|
if os.path.isdir(fullpath):
|
|
# TODO newer versions of vagrant provide `vagrant rsync`
|
|
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
|
|
subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
|
|
'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
|
|
fullpath + '/',
|
|
user + '@' + hostname + ':~/' + d + '/'])
|
|
|
|
# this file changes every time but should not be cached
|
|
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
|
|
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
|
|
|
|
print("Writing buildserver ID")
|
|
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
|
universal_newlines=True)
|
|
buildserverid = p.communicate()[0].strip()
|
|
print("...ID is " + buildserverid)
|
|
run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
|
|
|
|
print("Stopping build server VM")
|
|
v.halt()
|
|
|
|
print("Packaging")
|
|
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
|
|
if os.path.exists(boxfile):
|
|
os.remove(boxfile)
|
|
v.package(output=boxfile)
|
|
|
|
print("Adding box")
|
|
v.box_add('buildserver', boxfile, force=True)
|
|
|
|
os.remove(boxfile)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
finally:
|
|
if tail is not None:
|
|
tail.stop()
|