mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
99d0c55fe9
Add optional support for vagrant-cachier plugin Building the basebox is excruciating for people on slow connections. I'm particularly sensitive to this after living in Central America for awhile :) This won't affect anyone who hasn't installed the plugin. For those who do, it creates a persistent shared folder for each box (ie. testing23.box) and detects directories to cache between VM builds (apt, gems, pip, chef cache, etc.) (The only downside is that, for those following server setup does who are not aware what vagrant-cachier does, it might be unexpected that artifacts persist between vagrant destroys.) See merge request !25
281 lines
9.9 KiB
Python
Executable File
281 lines
9.9 KiB
Python
Executable File
#!/usr/bin/env python2
|
|
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import time
|
|
import hashlib
|
|
from optparse import OptionParser
|
|
|
|
|
|
def vagrant(params, cwd=None, printout=False):
|
|
"""Run vagrant.
|
|
|
|
:param: list of parameters to pass to vagrant
|
|
:cwd: directory to run in, or None for current directory
|
|
:printout: True to print output in realtime, False to just
|
|
return it
|
|
:returns: (ret, out) where ret is the return code, and out
|
|
is the stdout (and stderr) from vagrant
|
|
"""
|
|
p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
out = ''
|
|
if printout:
|
|
while True:
|
|
line = p.stdout.readline()
|
|
if len(line) == 0:
|
|
break
|
|
print line,
|
|
out += line
|
|
p.wait()
|
|
else:
|
|
out = p.communicate()[0]
|
|
return (p.returncode, out)
|
|
|
|
boxfile = 'buildserver.box'
|
|
serverdir = 'buildserver'
|
|
|
|
parser = OptionParser()
|
|
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
|
help="Spew out even more information than normal")
|
|
parser.add_option("-c", "--clean", action="store_true", default=False,
|
|
help="Build from scratch, rather than attempting to update the existing server")
|
|
options, args = parser.parse_args()
|
|
|
|
config = {}
|
|
execfile('makebs.config.py', config)
|
|
|
|
if not os.path.exists('makebuildserver') or not os.path.exists(serverdir):
|
|
print 'This must be run from the correct directory!'
|
|
sys.exit(1)
|
|
|
|
if os.path.exists(boxfile):
|
|
os.remove(boxfile)
|
|
|
|
if options.clean:
|
|
vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose)
|
|
|
|
# Update cached files.
|
|
cachedir = os.path.join('buildserver', 'cache')
|
|
if not os.path.exists(cachedir):
|
|
os.mkdir(cachedir)
|
|
|
|
cachefiles = [
|
|
('android-sdk_r24.3.4-linux.tgz',
|
|
'https://dl.google.com/android/android-sdk_r24.3.4-linux.tgz',
|
|
'886412375d8fe6e49a1583e57a8a36a47943666da681701ba9ad1ab7236e83ea'),
|
|
('gradle-1.4-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.4-bin.zip',
|
|
'cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e'),
|
|
('gradle-1.6-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.6-bin.zip',
|
|
'de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4'),
|
|
('gradle-1.7-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.7-bin.zip',
|
|
'360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30'),
|
|
('gradle-1.8-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.8-bin.zip',
|
|
'a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703'),
|
|
('gradle-1.9-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.9-bin.zip',
|
|
'097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9'),
|
|
('gradle-1.10-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.10-bin.zip',
|
|
'6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe'),
|
|
('gradle-1.11-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.11-bin.zip',
|
|
'07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091'),
|
|
('gradle-1.12-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-1.12-bin.zip',
|
|
'8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
|
|
('gradle-2.1-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.1-bin.zip',
|
|
'3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
|
|
('gradle-2.2.1-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
|
|
'420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
|
|
('gradle-2.3-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.3-bin.zip',
|
|
'010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774'),
|
|
('gradle-2.4-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.4-bin.zip',
|
|
'c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f'),
|
|
('gradle-2.5-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.5-bin.zip',
|
|
'3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555'),
|
|
('gradle-2.6-bin.zip',
|
|
'https://services.gradle.org/distributions/gradle-2.6-bin.zip',
|
|
'18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83'),
|
|
('Kivy-1.7.2.tar.gz',
|
|
'https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz',
|
|
'0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'),
|
|
]
|
|
|
|
if config['arch64']:
|
|
cachefiles.extend([
|
|
('android-ndk-r10e-linux-x86_64.bin',
|
|
'https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
|
|
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
|
|
('android-ndk-r9b-linux-x86_64.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
|
|
'8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
|
|
('android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
|
|
'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1')])
|
|
else:
|
|
cachefiles.extend([
|
|
('android-ndk-r10e-linux-x86.bin',
|
|
'https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86.bin',
|
|
'92b07d25aaad9b341a7f2b2a62402d508e948bf2dea3ee7b65a6aeb18bca7df5'),
|
|
('android-ndk-r9b-linux-x86.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86.tar.bz2',
|
|
'748104b829dd12afb2fdb3044634963abb24cdb0aad3b26030abe2e9e65bfc81'),
|
|
('android-ndk-r9b-linux-x86-legacy-toolchains.tar.bz2',
|
|
'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86-legacy-toolchains.tar.bz2',
|
|
'606aadf815ae28cc7b0154996247c70d609f111b14e44bcbcd6cad4c87fefb6f')])
|
|
wanted = []
|
|
|
|
|
|
def sha256_for_file(path):
|
|
with open(path, 'r') as f:
|
|
s = hashlib.sha256()
|
|
while True:
|
|
data = f.read(4096)
|
|
if not data:
|
|
break
|
|
s.update(data)
|
|
return s.hexdigest()
|
|
|
|
for f, src, shasum in cachefiles:
|
|
relpath = os.path.join(cachedir, f)
|
|
if not os.path.exists(relpath):
|
|
print "Downloading " + f + " to cache"
|
|
if subprocess.call(['wget', src], cwd=cachedir) != 0:
|
|
print "...download of " + f + " failed."
|
|
sys.exit(1)
|
|
if shasum:
|
|
v = sha256_for_file(relpath)
|
|
if v != shasum:
|
|
print "Invalid shasum of '" + v + "' detected for " + f
|
|
sys.exit(1)
|
|
else:
|
|
print "...shasum verified for " + f
|
|
|
|
wanted.append(f)
|
|
|
|
|
|
# Generate an appropriate Vagrantfile for the buildserver, based on our
|
|
# settings...
|
|
vagrantfile = """
|
|
Vagrant.configure("2") do |config|
|
|
|
|
if Vagrant.has_plugin?("vagrant-cachier")
|
|
config.cache.scope = :box
|
|
config.cache.auto_detect = false
|
|
config.cache.enable :apt
|
|
config.cache.enable :chef
|
|
end
|
|
|
|
config.vm.box = "{0}"
|
|
config.vm.box_url = "{1}"
|
|
|
|
config.vm.provider "virtualbox" do |v|
|
|
v.customize ["modifyvm", :id, "--memory", "{2}"]
|
|
v.customize ["modifyvm", :id, "--cpus", "{3}"]
|
|
end
|
|
|
|
config.vm.provision :shell, :path => "fixpaths.sh"
|
|
""".format(config['basebox'],
|
|
config['baseboxurl'],
|
|
config['memory'],
|
|
config.get('cpus', 1))
|
|
if 'aptproxy' in config and config['aptproxy']:
|
|
vagrantfile += """
|
|
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
|
|
""".format(config['aptproxy'])
|
|
vagrantfile += """
|
|
config.vm.provision :chef_solo do |chef|
|
|
chef.cookbooks_path = "cookbooks"
|
|
chef.log_level = :debug
|
|
chef.json = {
|
|
:settings => {
|
|
:sdk_loc => "/home/vagrant/android-sdk",
|
|
:ndk_loc => "/home/vagrant/android-ndk",
|
|
:user => "vagrant"
|
|
}
|
|
}
|
|
chef.add_recipe "fdroidbuild-general"
|
|
chef.add_recipe "android-sdk"
|
|
chef.add_recipe "android-ndk"
|
|
chef.add_recipe "gradle"
|
|
chef.add_recipe "kivy"
|
|
end
|
|
end
|
|
"""
|
|
|
|
# Check against the existing Vagrantfile, and if they differ, we need to
|
|
# create a new box:
|
|
vf = os.path.join(serverdir, 'Vagrantfile')
|
|
writevf = True
|
|
if os.path.exists(vf):
|
|
vagrant(['halt'], serverdir)
|
|
with open(vf, 'r') as f:
|
|
oldvf = f.read()
|
|
if oldvf != vagrantfile:
|
|
print "Server configuration has changed, rebuild from scratch is required"
|
|
vagrant(['destroy', '-f'], serverdir)
|
|
else:
|
|
print "Re-provisioning existing server"
|
|
writevf = False
|
|
else:
|
|
print "No existing server - building from scratch"
|
|
if writevf:
|
|
with open(vf, 'w') as f:
|
|
f.write(vagrantfile)
|
|
|
|
|
|
print "Configuring build server VM"
|
|
returncode, out = vagrant(['up'], serverdir, printout=True)
|
|
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
|
log.write(out)
|
|
if returncode != 0:
|
|
print "Failed to configure server"
|
|
sys.exit(1)
|
|
|
|
print "Writing buildserver ID"
|
|
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE)
|
|
buildserverid = p.communicate()[0].strip()
|
|
print "...ID is " + buildserverid
|
|
subprocess.call(
|
|
['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"'
|
|
.format(buildserverid)],
|
|
cwd=serverdir)
|
|
|
|
print "Stopping build server VM"
|
|
vagrant(['halt'], serverdir)
|
|
|
|
print "Waiting for build server VM to be finished"
|
|
ready = False
|
|
while not ready:
|
|
time.sleep(2)
|
|
returncode, out = vagrant(['status'], serverdir)
|
|
if returncode != 0:
|
|
print "Error while checking status"
|
|
sys.exit(1)
|
|
for line in out.splitlines():
|
|
if line.startswith("default"):
|
|
if line.find("poweroff") != -1:
|
|
ready = True
|
|
else:
|
|
print "Status: " + line
|
|
|
|
print "Packaging"
|
|
vagrant(['package', '--output', os.path.join('..', boxfile)], serverdir,
|
|
printout=options.verbose)
|
|
print "Adding box"
|
|
vagrant(['box', 'add', 'buildserver', boxfile, '-f'],
|
|
printout=options.verbose)
|
|
|
|
os.remove(boxfile)
|