mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
4b0a6ed29f
config.vm.box_url can be a list/tuple of URLs, which is useful to specific a locally cached copy. This is needed on slow connections, so that if it fails, the download of jessie32.box does not have to start from the beginning of the file again.
286 lines
10 KiB
Python
Executable File
286 lines
10 KiB
Python
Executable File
#!/usr/bin/env python2
|
|
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import time
|
|
import hashlib
|
|
from optparse import OptionParser
|
|
|
|
|
|
def vagrant(params, cwd=None, printout=False):
|
|
"""Run vagrant.
|
|
|
|
:param: list of parameters to pass to vagrant
|
|
:cwd: directory to run in, or None for current directory
|
|
:printout: True to print output in realtime, False to just
|
|
return it
|
|
:returns: (ret, out) where ret is the return code, and out
|
|
is the stdout (and stderr) from vagrant
|
|
"""
|
|
p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
out = ''
|
|
if printout:
|
|
while True:
|
|
line = p.stdout.readline()
|
|
if len(line) == 0:
|
|
break
|
|
print line,
|
|
out += line
|
|
p.wait()
|
|
else:
|
|
out = p.communicate()[0]
|
|
return (p.returncode, out)
|
|
|
|
boxfile = 'buildserver.box'
|
|
serverdir = 'buildserver'
|
|
|
|
parser = OptionParser()
|
|
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
|
help="Spew out even more information than normal")
|
|
parser.add_option("-c", "--clean", action="store_true", default=False,
|
|
help="Build from scratch, rather than attempting to update the existing server")
|
|
options, args = parser.parse_args()
|
|
|
|
config = {}
|
|
execfile('makebs.config.py', config)
|
|
|
|
if not os.path.exists('makebuildserver') or not os.path.exists(serverdir):
|
|
print 'This must be run from the correct directory!'
|
|
sys.exit(1)
|
|
|
|
if os.path.exists(boxfile):
|
|
os.remove(boxfile)
|
|
|
|
if options.clean:
|
|
vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose)
|
|
|
|
# Update cached files.
|
|
cachedir = os.path.join('buildserver', 'cache')
|
|
if not os.path.exists(cachedir):
|
|
os.mkdir(cachedir)
|
|
|
|
cachefiles = [
|
|
('android-sdk_r24.3.4-linux.tgz',
|
|
'https://dl.google.com/android/android-sdk_r24.3.4-linux.tgz',
|
|
'886412375d8fe6e49a1583e57a8a36a47943666da681701ba9ad1ab7236e83ea'),
|
|
('gradle-1.4-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.4-bin.zip',
|
|
'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
|
|
('gradle-1.6-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.6-bin.zip',
|
|
'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
|
|
('gradle-1.7-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.7-bin.zip',
|
|
'360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
|
|
('gradle-1.8-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.8-bin.zip',
|
|
'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
|
|
('gradle-1.9-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.9-bin.zip',
|
|
'097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
|
|
('gradle-1.10-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.10-bin.zip',
|
|
'6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
|
|
('gradle-1.11-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.11-bin.zip',
|
|
'07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
|
|
('gradle-1.12-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.12-bin.zip',
|
|
'8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
|
|
('gradle-2.1-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.1-bin.zip',
|
|
'3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
|
|
('gradle-2.2.1-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
|
|
'420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
|
|
('gradle-2.3-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.3-bin.zip',
|
|
'010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
|
|
('gradle-2.4-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.4-bin.zip',
|
|
'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
|
|
('gradle-2.5-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.5-bin.zip',
|
|
'3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
|
|
('gradle-2.6-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.6-bin.zip',
|
|
'18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
|
|
('Kivy-1.7.2.tar.gz',
|
|
'https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz',
|
|
'0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'),
|
|
]
|
|
|
|
if config['arch64']:
|
|
cachefiles.extend([
|
|
('android-ndk-r10e-linux-x86_64.bin',
|
|
'https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
|
|
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
|
|
('android-ndk-r9b-linux-x86_64.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
|
|
'8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
|
|
('android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
|
|
'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1')])
|
|
else:
|
|
cachefiles.extend([
|
|
('android-ndk-r10e-linux-x86.bin',
|
|
'https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86.bin',
|
|
'92b07d25aaad9b341a7f2b2a62402d508e948bf2dea3ee7b65a6aeb18bca7df5'),
|
|
('android-ndk-r9b-linux-x86.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86.tar.bz2',
|
|
'748104b829dd12afb2fdb3044634963abb24cdb0aad3b26030abe2e9e65bfc81'),
|
|
('android-ndk-r9b-linux-x86-legacy-toolchains.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86-legacy-toolchains.tar.bz2',
|
|
'606aadf815ae28cc7b0154996247c70d609f111b14e44bcbcd6cad4c87fefb6f')])
|
|
wanted = []
|
|
|
|
|
|
def sha256_for_file(path):
|
|
with open(path, 'r') as f:
|
|
s = hashlib.sha256()
|
|
while True:
|
|
data = f.read(4096)
|
|
if not data:
|
|
break
|
|
s.update(data)
|
|
return s.hexdigest()
|
|
|
|
for f, src, shasum in cachefiles:
|
|
relpath = os.path.join(cachedir, f)
|
|
if not os.path.exists(relpath):
|
|
print "Downloading " + f + " to cache"
|
|
if subprocess.call(['wget', src], cwd=cachedir) != 0:
|
|
print "...download of " + f + " failed."
|
|
sys.exit(1)
|
|
if shasum:
|
|
v = sha256_for_file(relpath)
|
|
if v != shasum:
|
|
print "Invalid shasum of '" + v + "' detected for " + f
|
|
sys.exit(1)
|
|
else:
|
|
print "...shasum verified for " + f
|
|
|
|
wanted.append(f)
|
|
|
|
# allow specifying a list/tuple that includes cached local copy
|
|
if type(config['baseboxurl']) in (list, tuple) or config['baseboxurl'][0] in ('(', '['):
|
|
baseboxurl = config['baseboxurl']
|
|
else:
|
|
baseboxurl = '"{0}"'.format(config['baseboxurl'])
|
|
|
|
# Generate an appropriate Vagrantfile for the buildserver, based on our
|
|
# settings...
|
|
vagrantfile = """
|
|
Vagrant.configure("2") do |config|
|
|
|
|
if Vagrant.has_plugin?("vagrant-cachier")
|
|
config.cache.scope = :box
|
|
config.cache.auto_detect = false
|
|
config.cache.enable :apt
|
|
config.cache.enable :chef
|
|
end
|
|
|
|
config.vm.box = "{0}"
|
|
config.vm.box_url = {1}
|
|
|
|
config.vm.provider "virtualbox" do |v|
|
|
v.customize ["modifyvm", :id, "--memory", "{2}"]
|
|
v.customize ["modifyvm", :id, "--cpus", "{3}"]
|
|
end
|
|
|
|
config.vm.provision :shell, :path => "fixpaths.sh"
|
|
""".format(config['basebox'],
|
|
baseboxurl,
|
|
config['memory'],
|
|
config.get('cpus', 1))
|
|
if 'aptproxy' in config and config['aptproxy']:
|
|
vagrantfile += """
|
|
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
|
|
""".format(config['aptproxy'])
|
|
vagrantfile += """
|
|
config.vm.provision :chef_solo do |chef|
|
|
chef.cookbooks_path = "cookbooks"
|
|
chef.log_level = :debug
|
|
chef.json = {
|
|
:settings => {
|
|
:sdk_loc => "/home/vagrant/android-sdk",
|
|
:ndk_loc => "/home/vagrant/android-ndk",
|
|
:user => "vagrant"
|
|
}
|
|
}
|
|
chef.add_recipe "fdroidbuild-general"
|
|
chef.add_recipe "android-sdk"
|
|
chef.add_recipe "android-ndk"
|
|
chef.add_recipe "gradle"
|
|
chef.add_recipe "kivy"
|
|
end
|
|
end
|
|
"""
|
|
|
|
# Check against the existing Vagrantfile, and if they differ, we need to
|
|
# create a new box:
|
|
vf = os.path.join(serverdir, 'Vagrantfile')
|
|
writevf = True
|
|
if os.path.exists(vf):
|
|
vagrant(['halt'], serverdir)
|
|
with open(vf, 'r') as f:
|
|
oldvf = f.read()
|
|
if oldvf != vagrantfile:
|
|
print "Server configuration has changed, rebuild from scratch is required"
|
|
vagrant(['destroy', '-f'], serverdir)
|
|
else:
|
|
print "Re-provisioning existing server"
|
|
writevf = False
|
|
else:
|
|
print "No existing server - building from scratch"
|
|
if writevf:
|
|
with open(vf, 'w') as f:
|
|
f.write(vagrantfile)
|
|
|
|
|
|
print "Configuring build server VM"
|
|
returncode, out = vagrant(['up'], serverdir, printout=True)
|
|
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
|
log.write(out)
|
|
if returncode != 0:
|
|
print "Failed to configure server"
|
|
sys.exit(1)
|
|
|
|
print "Writing buildserver ID"
|
|
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
|
|
buildserverid = p.communicate()[0].strip()
|
|
print "...ID is " + buildserverid
|
|
subprocess.call(
|
|
['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"'
|
|
.format(buildserverid)],
|
|
cwd=serverdir)
|
|
|
|
print "Stopping build server VM"
|
|
vagrant(['halt'], serverdir)
|
|
|
|
print "Waiting for build server VM to be finished"
|
|
ready = False
|
|
while not ready:
|
|
time.sleep(2)
|
|
returncode, out = vagrant(['status'], serverdir)
|
|
if returncode != 0:
|
|
print "Error while checking status"
|
|
sys.exit(1)
|
|
for line in out.splitlines():
|
|
if line.startswith("default"):
|
|
if line.find("poweroff") != -1:
|
|
ready = True
|
|
else:
|
|
print "Status: " + line
|
|
|
|
print "Packaging"
|
|
vagrant(['package', '--output', os.path.join('..', boxfile)], serverdir,
|
|
printout=options.verbose)
|
|
print "Adding box"
|
|
vagrant(['box', 'add', 'buildserver', boxfile, '-f'],
|
|
printout=options.verbose)
|
|
|
|
os.remove(boxfile)
|