mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-04 02:00:11 +02:00
Work in progress on integrating build server
This commit is contained in:
parent
d6e390afd6
commit
498e7d3c5f
1
.gitignore
vendored
1
.gitignore
vendored
@ -6,3 +6,4 @@ tmp/
|
|||||||
build_*/
|
build_*/
|
||||||
*~
|
*~
|
||||||
*.pyc
|
*.pyc
|
||||||
|
buildserver.box
|
||||||
|
17
README.buildserver
Normal file
17
README.buildserver
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
|
||||||
|
Setting up a build server:
|
||||||
|
|
||||||
|
1. Install VirtualBox, vagrant and vagrant-snap
|
||||||
|
2. In the buildserver directory, run 'vagrant up'. Will take a long time.
|
||||||
|
3. Log in with 'vagrant ssh'
|
||||||
|
4. Check it all looks ok, then 'sudo shutdown -h now'
|
||||||
|
5. Back in the main directory, run 'VBoxManage listvms' look for
|
||||||
|
buildserver_xxxx
|
||||||
|
6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'.
|
||||||
|
Will take a while.
|
||||||
|
7. You should now have a new 'buildserver.box'
|
||||||
|
|
||||||
|
You should now be able to use the --server option on build.py and builds will
|
||||||
|
take place in the clean, secure, isolated environment of a fresh virtual
|
||||||
|
machine for each app built.
|
||||||
|
|
87
build.py
87
build.py
@ -24,6 +24,7 @@ import subprocess
|
|||||||
import re
|
import re
|
||||||
import zipfile
|
import zipfile
|
||||||
import tarfile
|
import tarfile
|
||||||
|
import traceback
|
||||||
from xml.dom.minidom import Document
|
from xml.dom.minidom import Document
|
||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
|
|
||||||
@ -44,8 +45,18 @@ parser.add_option("-s", "--stop", action="store_true", default=False,
|
|||||||
help="Make the build stop on exceptions")
|
help="Make the build stop on exceptions")
|
||||||
parser.add_option("-t", "--test", action="store_true", default=False,
|
parser.add_option("-t", "--test", action="store_true", default=False,
|
||||||
help="Test mode - put output in the tmp directory only.")
|
help="Test mode - put output in the tmp directory only.")
|
||||||
|
parser.add_option("--server", action="store_true", default=False,
|
||||||
|
help="Use build server")
|
||||||
|
parser.add_option("--on-server", action="store_true", default=False,
|
||||||
|
help="Specify that we're running on the build server")
|
||||||
|
parser.add_option("-f", "--force", action="store_true", default=False,
|
||||||
|
help="Force build of disabled app. Only allowed in test mode.")
|
||||||
(options, args) = parser.parse_args()
|
(options, args) = parser.parse_args()
|
||||||
|
|
||||||
|
if options.force and not options.test:
|
||||||
|
print "Force is only allowed in test mode"
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
# Get all apps...
|
# Get all apps...
|
||||||
apps = common.read_metadata(options.verbose)
|
apps = common.read_metadata(options.verbose)
|
||||||
|
|
||||||
@ -84,7 +95,7 @@ for app in apps:
|
|||||||
if options.package and options.package != app['id']:
|
if options.package and options.package != app['id']:
|
||||||
# Silent skip...
|
# Silent skip...
|
||||||
pass
|
pass
|
||||||
elif app['Disabled']:
|
elif app['Disabled'] and not options.force:
|
||||||
if options.verbose:
|
if options.verbose:
|
||||||
print "Skipping %s: disabled" % app['id']
|
print "Skipping %s: disabled" % app['id']
|
||||||
elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
|
elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
|
||||||
@ -107,7 +118,7 @@ for app in apps:
|
|||||||
dest_repo = os.path.join(repo_dir, app['id'] + '_' +
|
dest_repo = os.path.join(repo_dir, app['id'] + '_' +
|
||||||
thisbuild['vercode'] + '.apk')
|
thisbuild['vercode'] + '.apk')
|
||||||
|
|
||||||
if os.path.exists(dest) or os.path.exists(dest_repo):
|
if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
|
||||||
if options.verbose:
|
if options.verbose:
|
||||||
print "..version " + thisbuild['version'] + " already exists"
|
print "..version " + thisbuild['version'] + " already exists"
|
||||||
elif thisbuild['commit'].startswith('!'):
|
elif thisbuild['commit'].startswith('!'):
|
||||||
@ -121,6 +132,74 @@ for app in apps:
|
|||||||
mstart = 'Building version '
|
mstart = 'Building version '
|
||||||
print mstart + thisbuild['version'] + ' of ' + app['id']
|
print mstart + thisbuild['version'] + ' of ' + app['id']
|
||||||
|
|
||||||
|
if options.server:
|
||||||
|
|
||||||
|
import paramiko
|
||||||
|
|
||||||
|
# Start up the virtual maachine...
|
||||||
|
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
|
||||||
|
# Not a very helpful message yet!
|
||||||
|
raise BuildException("Failed to set up build server")
|
||||||
|
# Get SSH configuration settings for us to connect...
|
||||||
|
subprocess.call('vagrant ssh-config >sshconfig',
|
||||||
|
cwd='builder', shell=True)
|
||||||
|
vagranthost = 'default' # Host in ssh config file
|
||||||
|
|
||||||
|
# Load and parse the SSH config...
|
||||||
|
sshconfig = paramiko.SSHConfig()
|
||||||
|
sshconfig.parse('builder/sshconfig')
|
||||||
|
sshconfig = sshconfig.lookup(vagranthost)
|
||||||
|
|
||||||
|
# Open SSH connection...
|
||||||
|
ssh = paramiko.SSHClient()
|
||||||
|
ssh.set_missing_host_key_policy(paramiko.AcceptPolicy)
|
||||||
|
ssh.connect(sshconfig['HostName'], username=sshconfig['Username'],
|
||||||
|
port=sshconfig['Port'], timeout=10, look_for_keys=False,
|
||||||
|
key_filename=sshconfig['IdentityFile'])
|
||||||
|
|
||||||
|
# Get an SFTP connection...
|
||||||
|
ftp = ssh.open_sftp()
|
||||||
|
ftp.get_channel().settimeout(15)
|
||||||
|
|
||||||
|
# Put all the necessary files in place...
|
||||||
|
ftp.chdir('/home/vagrant')
|
||||||
|
ftp.put('build.py', 'build.py')
|
||||||
|
ftp.put('common.py', 'common.py')
|
||||||
|
ftp.put('config.buildserver.py', 'config.py')
|
||||||
|
ftp.mkdir('build')
|
||||||
|
ftp.chdir('build')
|
||||||
|
ftp.mkdir('extlib')
|
||||||
|
def send_dir(path):
|
||||||
|
lastdir = path
|
||||||
|
for r, d, f in os.walk(base):
|
||||||
|
while lastdir != os.path.commonprefix([lastdir, root]):
|
||||||
|
ftp.chdir('..')
|
||||||
|
lastdir = os.path.split(lastdir)[0]
|
||||||
|
lastdir = r
|
||||||
|
for ff in f:
|
||||||
|
ftp.put(os.path.join(r, ff), ff)
|
||||||
|
ftp.send_dir(app['id'])
|
||||||
|
# TODO: send relevant extlib directories too
|
||||||
|
ftp.chdir('/home/vagrant')
|
||||||
|
|
||||||
|
# Execute the build script...
|
||||||
|
ssh.exec_command('python build.py --on-server -p ' +
|
||||||
|
app['id'])
|
||||||
|
|
||||||
|
# Retrieve the built files...
|
||||||
|
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
|
||||||
|
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
|
||||||
|
ftp.chdir('unsigned')
|
||||||
|
ftp.get(apkfile, os.path.join(output_dir, apkfile))
|
||||||
|
ftp.get(tarball, os.path.join(output_dir, tarball))
|
||||||
|
|
||||||
|
# Get rid of the virtual machine...
|
||||||
|
if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
|
||||||
|
# Not a very helpful message yet!
|
||||||
|
raise BuildException("Failed to destroy")
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
# Prepare the source code...
|
# Prepare the source code...
|
||||||
root_dir = common.prepare_source(vcs, app, thisbuild,
|
root_dir = common.prepare_source(vcs, app, thisbuild,
|
||||||
build_dir, extlib_dir, sdk_path, ndk_path,
|
build_dir, extlib_dir, sdk_path, ndk_path,
|
||||||
@ -128,7 +207,7 @@ for app in apps:
|
|||||||
refreshed_source = True
|
refreshed_source = True
|
||||||
|
|
||||||
# Scan before building...
|
# Scan before building...
|
||||||
buildprobs = common.scan_source(build_dir)
|
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
|
||||||
if len(buildprobs) > 0:
|
if len(buildprobs) > 0:
|
||||||
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
|
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
|
||||||
for problem in buildprobs:
|
for problem in buildprobs:
|
||||||
@ -264,7 +343,7 @@ for app in apps:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
if options.stop:
|
if options.stop:
|
||||||
raise
|
raise
|
||||||
print "Could not build app %s due to unknown error: %s" % (app['id'], e)
|
print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
|
||||||
failed_apps[app['id']] = e
|
failed_apps[app['id']] = e
|
||||||
|
|
||||||
for app in build_succeeded:
|
for app in build_succeeded:
|
||||||
|
1
builder/.gitignore
vendored
Normal file
1
builder/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
sshconfig
|
8
builder/Vagrantfile
vendored
Normal file
8
builder/Vagrantfile
vendored
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
Vagrant::Config.run do |config|
|
||||||
|
|
||||||
|
config.vm.box = "buildserver"
|
||||||
|
config.vm.box_url = "../buildserver.box"
|
||||||
|
|
||||||
|
config.vm.customize ["modifyvm", :id, "--memory", "768"]
|
||||||
|
|
||||||
|
end
|
1
buildserver/.gitignore
vendored
Normal file
1
buildserver/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
.vagrant
|
26
buildserver/Vagrantfile
vendored
Normal file
26
buildserver/Vagrantfile
vendored
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
Vagrant::Config.run do |config|
|
||||||
|
|
||||||
|
config.vm.box = "debian6-32"
|
||||||
|
config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box"
|
||||||
|
|
||||||
|
config.vm.customize ["modifyvm", :id, "--memory", "1024"]
|
||||||
|
|
||||||
|
config.vm.provision :shell, :path => "fixpaths.sh"
|
||||||
|
# Set apt proxy - remove, or adjust this, accordingly!
|
||||||
|
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://thurlow:3142\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
|
||||||
|
|
||||||
|
config.vm.provision :chef_solo do |chef|
|
||||||
|
chef.cookbooks_path = "cookbooks"
|
||||||
|
chef.log_level = :debug
|
||||||
|
chef.json = {
|
||||||
|
:settings => {
|
||||||
|
:sdk_loc => "/home/vagrant/android-sdk",
|
||||||
|
:ndk_loc => "/home/vagrant/android-ndk",
|
||||||
|
:user => "vagrant"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chef.add_recipe "android-sdk"
|
||||||
|
chef.add_recipe "android-ndk"
|
||||||
|
chef.add_recipe "fdroidbuild-general"
|
||||||
|
end
|
||||||
|
end
|
17
buildserver/cookbooks/android-ndk/recipes/default.rb
Normal file
17
buildserver/cookbooks/android-ndk/recipes/default.rb
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
|
||||||
|
ndk_loc = node[:settings][:ndk_loc]
|
||||||
|
|
||||||
|
script "setup-android-ndk" do
|
||||||
|
interpreter "bash"
|
||||||
|
user node[:settings][:user]
|
||||||
|
cwd "/tmp"
|
||||||
|
code "
|
||||||
|
wget http://dl.google.com/android/ndk/android-ndk-r7-linux-x86.tar.bz2
|
||||||
|
tar jxvf android-ndk-r7-linux-x86.tar.bz2
|
||||||
|
mv android-ndk-r7 #{ndk_loc}
|
||||||
|
"
|
||||||
|
not_if do
|
||||||
|
File.exists?("#{ndk_loc}")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
34
buildserver/cookbooks/android-sdk/recipes/default.rb
Normal file
34
buildserver/cookbooks/android-sdk/recipes/default.rb
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
%w{openjdk-6-jdk}.each do |pkg|
|
||||||
|
package pkg do
|
||||||
|
action :install
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
sdk_loc = node[:settings][:sdk_loc]
|
||||||
|
user = node[:settings][:user]
|
||||||
|
|
||||||
|
script "setup-android-sdk" do
|
||||||
|
interpreter "bash"
|
||||||
|
user user
|
||||||
|
cwd "/tmp"
|
||||||
|
code "
|
||||||
|
wget http://dl.google.com/android/android-sdk_r16-linux.tgz
|
||||||
|
tar zxvf android-sdk_r16-linux.tgz
|
||||||
|
mv android-sdk-linux #{sdk_loc}
|
||||||
|
rm android-sdk_r16-linux.tgz
|
||||||
|
#{sdk_loc}/tools/android update sdk --no-ui -t platform-tool
|
||||||
|
#{sdk_loc}/tools/android update sdk --no-ui -t platform
|
||||||
|
#{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool
|
||||||
|
"
|
||||||
|
not_if do
|
||||||
|
File.exists?("#{sdk_loc}")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
execute "add-android-sdk-path" do
|
||||||
|
user user
|
||||||
|
path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools"
|
||||||
|
command "echo \"export PATH=\\$PATH:#{path}\" >> /home/#{user}/.bashrc"
|
||||||
|
not_if "grep #{sdk_loc} /home/#{user}/.bashrc"
|
||||||
|
end
|
||||||
|
|
@ -0,0 +1,7 @@
|
|||||||
|
|
||||||
|
%w{ant ant-contrib maven2 javacc python}.each do |pkg|
|
||||||
|
package pkg do
|
||||||
|
action :install
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
23
buildserver/fixpaths.sh
Normal file
23
buildserver/fixpaths.sh
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
fixit()
|
||||||
|
{
|
||||||
|
#Fix sudoers so the PATH gets passed through, otherwise chef
|
||||||
|
#provisioning doesn't work.
|
||||||
|
if [ -z "$1" ]; then
|
||||||
|
export EDITOR=$0 && sudo -E visudo
|
||||||
|
else
|
||||||
|
echo "Fix sudoers"
|
||||||
|
echo "Defaults exempt_group=admin" >> $1
|
||||||
|
fi
|
||||||
|
#Stick the gems bin onto root's path as well.
|
||||||
|
sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc
|
||||||
|
# Restart sudo so it gets the changes straight away
|
||||||
|
sudo /etc/init.d/sudo restart
|
||||||
|
}
|
||||||
|
|
||||||
|
sudo grep "exempt_group" /etc/sudoers -q
|
||||||
|
if [ "$?" -eq "1" ]; then
|
||||||
|
fixit
|
||||||
|
fi
|
||||||
|
|
@ -892,7 +892,7 @@ def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, j
|
|||||||
|
|
||||||
# Scan the source code in the given directory (and all subdirectories)
|
# Scan the source code in the given directory (and all subdirectories)
|
||||||
# and return a list of potential problems.
|
# and return a list of potential problems.
|
||||||
def scan_source(source_dir):
|
def scan_source(build_dir, root_dir, thisbuild):
|
||||||
|
|
||||||
problems = []
|
problems = []
|
||||||
|
|
||||||
|
4
config.buildserver.py
Normal file
4
config.buildserver.py
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
aapt_path = "/home/vagrant/android-sdk/platform-tools/aapt"
|
||||||
|
sdk_path = "/home/vagrant/android-sdk"
|
||||||
|
ndk_path = "/home/vagrant/android-ndk"
|
||||||
|
javacc_path = "/usr/share/java"
|
@ -93,7 +93,7 @@ for app in apps:
|
|||||||
refreshed_source = True
|
refreshed_source = True
|
||||||
|
|
||||||
# Do the scan...
|
# Do the scan...
|
||||||
buildprobs = common.scan_source(build_dir)
|
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
|
||||||
for problem in buildprobs:
|
for problem in buildprobs:
|
||||||
problems.append(problem +
|
problems.append(problem +
|
||||||
' in ' + app['id'] + ' ' + thisbuild['version'])
|
' in ' + app['id'] + ' ' + thisbuild['version'])
|
||||||
|
Loading…
Reference in New Issue
Block a user