mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-05 18:50:09 +02:00
Much better build server provisioning
Including selection of correct ndk architecture, caching and performance improvements, better configurability, etc.
This commit is contained in:
parent
0985a6d4f4
commit
cb16a63409
2
buildserver/.gitignore
vendored
2
buildserver/.gitignore
vendored
@ -1,2 +1,4 @@
|
|||||||
.vagrant
|
.vagrant
|
||||||
up.log
|
up.log
|
||||||
|
cache/
|
||||||
|
Vagrantfile
|
||||||
|
26
buildserver/Vagrantfile
vendored
26
buildserver/Vagrantfile
vendored
@ -1,26 +0,0 @@
|
|||||||
Vagrant::Config.run do |config|
|
|
||||||
|
|
||||||
config.vm.box = "raring64"
|
|
||||||
config.vm.box_url = "/shares/software/OS and Boot/raring64.box"
|
|
||||||
|
|
||||||
config.vm.customize ["modifyvm", :id, "--memory", "3584"]
|
|
||||||
|
|
||||||
config.vm.provision :shell, :path => "fixpaths.sh"
|
|
||||||
# Set apt proxy - remove, or adjust this, accordingly!
|
|
||||||
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://192.168.0.19:8000\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
|
|
||||||
|
|
||||||
config.vm.provision :chef_solo do |chef|
|
|
||||||
chef.cookbooks_path = "cookbooks"
|
|
||||||
chef.log_level = :debug
|
|
||||||
chef.json = {
|
|
||||||
:settings => {
|
|
||||||
:sdk_loc => "/home/vagrant/android-sdk",
|
|
||||||
:ndk_loc => "/home/vagrant/android-ndk",
|
|
||||||
:user => "vagrant"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
chef.add_recipe "fdroidbuild-general"
|
|
||||||
chef.add_recipe "android-sdk"
|
|
||||||
chef.add_recipe "android-ndk"
|
|
||||||
end
|
|
||||||
end
|
|
@ -7,7 +7,12 @@ script "setup-android-ndk" do
|
|||||||
user node[:settings][:user]
|
user node[:settings][:user]
|
||||||
cwd "/tmp"
|
cwd "/tmp"
|
||||||
code "
|
code "
|
||||||
tar jxvf /vagrant/cache/android-ndk-r8e-linux-x64.tar.bz2
|
if [ `uname -m` == 'x86_64' ] ; then
|
||||||
|
SUFFIX = '_64'
|
||||||
|
else
|
||||||
|
SUFFIX = ''
|
||||||
|
fi
|
||||||
|
tar jxvf /vagrant/cache/android-ndk-r8e-linux-x86$SUFFIX.tar.bz2
|
||||||
mv android-ndk-r8e #{ndk_loc}
|
mv android-ndk-r8e #{ndk_loc}
|
||||||
"
|
"
|
||||||
not_if do
|
not_if do
|
||||||
|
@ -37,10 +37,16 @@ end
|
|||||||
user user
|
user user
|
||||||
cwd "/tmp"
|
cwd "/tmp"
|
||||||
code "
|
code "
|
||||||
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
if [ -f /vagrant/cache/platforms/#{sdk}.tar.gz ] ; then
|
||||||
|
echo Installing from cache
|
||||||
|
tar -C #{sdk_loc}/platforms -z -x -f /vagrant/cache/platforms/#{sdk}.tar.gz
|
||||||
|
else
|
||||||
|
echo Installing via 'android'
|
||||||
|
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
||||||
y
|
y
|
||||||
|
|
||||||
X
|
X
|
||||||
|
fi
|
||||||
"
|
"
|
||||||
not_if "test -d #{sdk_loc}/platforms/#{sdk}"
|
not_if "test -d #{sdk_loc}/platforms/#{sdk}"
|
||||||
end
|
end
|
||||||
@ -54,10 +60,16 @@ end
|
|||||||
user user
|
user user
|
||||||
cwd "/tmp"
|
cwd "/tmp"
|
||||||
code "
|
code "
|
||||||
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
if [ -f /vagrant/cache/add-ons/#{sdk}.tar.gz ] ; then
|
||||||
|
echo Installing from cache
|
||||||
|
tar -C #{sdk_loc}/add-ons -z -x -f /vagrant/cache/add-ons/#{sdk}.tar.gz
|
||||||
|
else
|
||||||
|
echo Installing via 'android'
|
||||||
|
#{sdk_loc}/tools/android update sdk --no-ui -a -t #{sdk} <<X
|
||||||
y
|
y
|
||||||
|
|
||||||
X
|
X
|
||||||
|
fi
|
||||||
"
|
"
|
||||||
|
|
||||||
not_if "test -d #{sdk_loc}/add-ons/#{sdk}"
|
not_if "test -d #{sdk_loc}/add-ons/#{sdk}"
|
||||||
|
@ -5,11 +5,40 @@ import sys
|
|||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
|
|
||||||
def vagrant(params, cwd=None):
|
|
||||||
|
# Settings, which need to be moved elsewhere ultimately...
|
||||||
|
basebox = "raring32"
|
||||||
|
baseboxurl = "/shares/software/OS and Boot/raring32.box"
|
||||||
|
memory = 3584
|
||||||
|
aptproxy = "http://192.168.0.19:8000"
|
||||||
|
arch64 = False
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def vagrant(params, cwd=None, printout=False):
|
||||||
|
"""Run vagrant.
|
||||||
|
|
||||||
|
:param: list of parameters to pass to vagrant
|
||||||
|
:cwd: directory to run in, or None for current directory
|
||||||
|
:printout: True to print output in realtime, False to just
|
||||||
|
return it
|
||||||
|
:returns: (ret, out) where ret is the return code, and out
|
||||||
|
is the stdout (and stderr) from vagrant
|
||||||
|
"""
|
||||||
p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
|
p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
out, err = p.communicate()
|
out = ''
|
||||||
return (p.returncode, out, err)
|
if printout:
|
||||||
|
while True:
|
||||||
|
line = p.stdout.readline()
|
||||||
|
if len(line) == 0:
|
||||||
|
break
|
||||||
|
print line,
|
||||||
|
out += line
|
||||||
|
p.wait()
|
||||||
|
else:
|
||||||
|
out = p.communicate()[0]
|
||||||
|
return (p.returncode, out)
|
||||||
|
|
||||||
boxfile = 'buildserver.box'
|
boxfile = 'buildserver.box'
|
||||||
serverdir = 'buildserver'
|
serverdir = 'buildserver'
|
||||||
@ -28,35 +57,84 @@ if not os.path.exists(cachedir):
|
|||||||
os.mkdir(cachedir)
|
os.mkdir(cachedir)
|
||||||
cachefiles = [
|
cachefiles = [
|
||||||
('android-sdk_r21.0.1-linux.tgz',
|
('android-sdk_r21.0.1-linux.tgz',
|
||||||
'http://dl.google.com/android/android-sdk_r21.0.1-linux.tgz',
|
'http://dl.google.com/android/android-sdk_r21.0.1-linux.tgz')]
|
||||||
'cookbooks/recipes/android-sdk/default.rb'),
|
if arch64:
|
||||||
('android-ndk-r8e-linux-x64.tar.bz2',
|
cachefiles.extend([
|
||||||
'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x64.tar.bz2',
|
('android-ndk-r8e-linux-x86_64.tar.bz2',
|
||||||
'cookbooks/recipes/android-ndk/default.rb')
|
'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x86_64.tar.bz2')])
|
||||||
]
|
else:
|
||||||
|
cachefiles.extend([
|
||||||
|
('android-ndk-r8e-linux-x86.tar.bz2',
|
||||||
|
'http://dl.google.com/android/ndk/android-ndk-r8e-linux-x86.tar.bz2')])
|
||||||
wanted = []
|
wanted = []
|
||||||
for f, src, check in cachefiles:
|
for f, src in cachefiles:
|
||||||
if subprocess.call('grep ' + f + ' ' + check) != 0:
|
|
||||||
print "Cache mismatch - " + f + " is not mentioned in " + check
|
|
||||||
sys.exit(1)
|
|
||||||
if not os.path.exists(os.path.join(cachedir, f)):
|
if not os.path.exists(os.path.join(cachedir, f)):
|
||||||
print "Downloading " + f + " to cache"
|
print "Downloading " + f + " to cache"
|
||||||
if subprocess.call('wget ' + src, cwd=cachedir) != 0:
|
if subprocess.call(['wget', src], cwd=cachedir) != 0:
|
||||||
print "...download of " + f + " failed."
|
print "...download of " + f + " failed."
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
wanted.append(f)
|
wanted.append(f)
|
||||||
for f in os.listdir(cachedir):
|
|
||||||
if not f in wanted:
|
|
||||||
print "Removing unwanted cache file " + f
|
|
||||||
os.remove(os.path.join(cachedir, f))
|
|
||||||
|
|
||||||
|
|
||||||
vagrant(['halt'], serverdir)
|
# Generate an appropriate Vagrantfile for the buildserver, based on our
|
||||||
|
# settings...
|
||||||
|
vagrantfile = """
|
||||||
|
Vagrant::Config.run do |config|
|
||||||
|
|
||||||
|
config.vm.box = "{0}"
|
||||||
|
config.vm.box_url = "{1}"
|
||||||
|
|
||||||
|
config.vm.customize ["modifyvm", :id, "--memory", "{2}"]
|
||||||
|
|
||||||
|
config.vm.provision :shell, :path => "fixpaths.sh"
|
||||||
|
""".format(basebox, baseboxurl, memory)
|
||||||
|
if aptproxy:
|
||||||
|
vagrantfile += """
|
||||||
|
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
|
||||||
|
""".format(aptproxy)
|
||||||
|
vagrantfile += """
|
||||||
|
config.vm.provision :chef_solo do |chef|
|
||||||
|
chef.cookbooks_path = "cookbooks"
|
||||||
|
chef.log_level = :debug
|
||||||
|
chef.json = {
|
||||||
|
:settings => {
|
||||||
|
:sdk_loc => "/home/vagrant/android-sdk",
|
||||||
|
:ndk_loc => "/home/vagrant/android-ndk",
|
||||||
|
:user => "vagrant"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chef.add_recipe "fdroidbuild-general"
|
||||||
|
chef.add_recipe "android-sdk"
|
||||||
|
chef.add_recipe "android-ndk"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Check against the existing Vagrantfile, and if they differ, we need to
|
||||||
|
# create a new box:
|
||||||
|
vf = os.path.join(serverdir, 'Vagrantfile')
|
||||||
|
writevf = True
|
||||||
|
if os.path.exists(vf):
|
||||||
|
vagrant(['halt'], serverdir)
|
||||||
|
with open(vf, 'r') as f:
|
||||||
|
oldvf = f.read()
|
||||||
|
if oldvf != vagrantfile:
|
||||||
|
print "Server configuration has changed, rebuild from scratch is required"
|
||||||
|
vagrant(['destroy', '-f'], serverdir)
|
||||||
|
else:
|
||||||
|
print "Re-provisioning existing server"
|
||||||
|
writevf = False
|
||||||
|
else:
|
||||||
|
print "No existing server - building from scratch"
|
||||||
|
if writevf:
|
||||||
|
with open(vf, 'w') as f:
|
||||||
|
f.write(vagrantfile)
|
||||||
|
|
||||||
|
|
||||||
print "Configuring build server VM"
|
print "Configuring build server VM"
|
||||||
returncode, out, err = vagrant(['up'], serverdir)
|
returncode, out = vagrant(['up'], serverdir, printout=True)
|
||||||
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
||||||
log.write('==stdout==\n' + out + '\n\n')
|
log.write(out)
|
||||||
log.write('==stderr==\n' + err + '\n\n')
|
|
||||||
if returncode != 0:
|
if returncode != 0:
|
||||||
print "Failed to configure server"
|
print "Failed to configure server"
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
@ -67,7 +145,7 @@ print "Waiting for build server VM to be finished"
|
|||||||
ready = False
|
ready = False
|
||||||
while not ready:
|
while not ready:
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
returncode, out, err = vagrant(['status'], serverdir)
|
returncode, out = vagrant(['status'], serverdir)
|
||||||
if returncode != 0:
|
if returncode != 0:
|
||||||
print "Error while checking status"
|
print "Error while checking status"
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
Loading…
Reference in New Issue
Block a user