kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-20 13:50:12 +01:00
Code Activity

Work in progress on integrating build server

Browse Source
This commit is contained in:
Ciaran Gultnieks 2012-02-03 16:01:35 +00:00
parent d6e390afd6
commit 498e7d3c5f
14 changed files with 334 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -6,3 +6,4 @@ tmp/
build_*/ build_*/
*~ *~
*.pyc *.pyc
buildserver.box

17
README.buildserver Normal file
View File

@ -0,0 +1,17 @@
Setting up a build server:
1. Install VirtualBox, vagrant and vagrant-snap
2. In the buildserver directory, run 'vagrant up'. Will take a long time.
3. Log in with 'vagrant ssh'
4. Check it all looks ok, then 'sudo shutdown -h now'
5. Back in the main directory, run 'VBoxManage listvms' look for
buildserver_xxxx
6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'.
Will take a while.
7. You should now have a new 'buildserver.box'
You should now be able to use the --server option on build.py and builds will
take place in the clean, secure, isolated environment of a fresh virtual
machine for each app built.

307
build.py
View File

@ -24,6 +24,7 @@ import subprocess
import re import re
import zipfile import zipfile
import tarfile import tarfile
import traceback
from xml.dom.minidom import Document from xml.dom.minidom import Document
from optparse import OptionParser from optparse import OptionParser
@ -44,8 +45,18 @@ parser.add_option("-s", "--stop", action="store_true", default=False,
help="Make the build stop on exceptions") help="Make the build stop on exceptions")
parser.add_option("-t", "--test", action="store_true", default=False, parser.add_option("-t", "--test", action="store_true", default=False,
help="Test mode - put output in the tmp directory only.") help="Test mode - put output in the tmp directory only.")
parser.add_option("--server", action="store_true", default=False,
help="Use build server")
parser.add_option("--on-server", action="store_true", default=False,
help="Specify that we're running on the build server")
parser.add_option("-f", "--force", action="store_true", default=False,
help="Force build of disabled app. Only allowed in test mode.")
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
if options.force and not options.test:
print "Force is only allowed in test mode"
sys.exit(1)
# Get all apps... # Get all apps...
apps = common.read_metadata(options.verbose) apps = common.read_metadata(options.verbose)
@ -84,7 +95,7 @@ for app in apps:
if options.package and options.package != app['id']: if options.package and options.package != app['id']:
# Silent skip... # Silent skip...
pass pass
elif app['Disabled']: elif app['Disabled'] and not options.force:
if options.verbose: if options.verbose:
print "Skipping %s: disabled" % app['id'] print "Skipping %s: disabled" % app['id']
elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0: elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
@ -107,7 +118,7 @@ for app in apps:
dest_repo = os.path.join(repo_dir, app['id'] + '_' + dest_repo = os.path.join(repo_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk') thisbuild['vercode'] + '.apk')
if os.path.exists(dest) or os.path.exists(dest_repo): if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
if options.verbose: if options.verbose:
print "..version " + thisbuild['version'] + " already exists" print "..version " + thisbuild['version'] + " already exists"
elif thisbuild['commit'].startswith('!'): elif thisbuild['commit'].startswith('!'):
@ -121,131 +132,199 @@ for app in apps:
mstart = 'Building version ' mstart = 'Building version '
print mstart + thisbuild['version'] + ' of ' + app['id'] print mstart + thisbuild['version'] + ' of ' + app['id']
# Prepare the source code... if options.server:
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path,
javacc_path, not refreshed_source)
refreshed_source = True
# Scan before building... import paramiko
buildprobs = common.scan_source(build_dir)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
# Build the source tarball right before we build the release... # Start up the virtual maachine...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
tarball = tarfile.open(os.path.join(tmp_dir, # Not a very helpful message yet!
tarname + '.tar.gz'), "w:gz") raise BuildException("Failed to set up build server")
def tarexc(f): # Get SSH configuration settings for us to connect...
if f in ['.svn', '.git', '.hg', '.bzr']: subprocess.call('vagrant ssh-config >sshconfig',
return True cwd='builder', shell=True)
return False vagranthost = 'default' # Host in ssh config file
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Build native stuff if required... # Load and parse the SSH config...
if thisbuild.get('buildjni', 'no') == 'yes': sshconfig = paramiko.SSHConfig()
ndkbuild = os.path.join(ndk_path, "ndk-build") sshconfig.parse('builder/sshconfig')
p = subprocess.Popen([ndkbuild], cwd=root_dir, sshconfig = sshconfig.lookup(vagranthost)
stdout=subprocess.PIPE)
output = p.communicate()[0] # Open SSH connection...
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AcceptPolicy)
ssh.connect(sshconfig['HostName'], username=sshconfig['Username'],
port=sshconfig['Port'], timeout=10, look_for_keys=False,
key_filename=sshconfig['IdentityFile'])
# Get an SFTP connection...
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir('/home/vagrant')
ftp.put('build.py', 'build.py')
ftp.put('common.py', 'common.py')
ftp.put('config.buildserver.py', 'config.py')
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
def send_dir(path):
lastdir = path
for r, d, f in os.walk(base):
while lastdir != os.path.commonprefix([lastdir, root]):
ftp.chdir('..')
lastdir = os.path.split(lastdir)[0]
lastdir = r
for ff in f:
ftp.put(os.path.join(r, ff), ff)
ftp.send_dir(app['id'])
# TODO: send relevant extlib directories too
ftp.chdir('/home/vagrant')
# Execute the build script...
ssh.exec_command('python build.py --on-server -p ' +
app['id'])
# Retrieve the built files...
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
ftp.chdir('unsigned')
ftp.get(apkfile, os.path.join(output_dir, apkfile))
ftp.get(tarball, os.path.join(output_dir, tarball))
# Get rid of the virtual machine...
if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
# Not a very helpful message yet!
raise BuildException("Failed to destroy")
else:
# Prepare the source code...
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path,
javacc_path, not refreshed_source)
refreshed_source = True
# Scan before building...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
# Build the source tarball right before we build the release...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
tarball = tarfile.open(os.path.join(tmp_dir,
tarname + '.tar.gz'), "w:gz")
def tarexc(f):
if f in ['.svn', '.git', '.hg', '.bzr']:
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Build native stuff if required...
if thisbuild.get('buildjni', 'no') == 'yes':
ndkbuild = os.path.join(ndk_path, "ndk-build")
p = subprocess.Popen([ndkbuild], cwd=root_dir,
stdout=subprocess.PIPE)
output = p.communicate()[0]
if p.returncode != 0:
print output
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
elif options.verbose:
print output
# Build the release...
if thisbuild.has_key('maven'):
p = subprocess.Popen(['mvn', 'clean', 'install',
'-Dandroid.sdk.path=' + sdk_path],
cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
if thisbuild.has_key('antcommand'):
antcommand = thisbuild['antcommand']
else:
antcommand = 'release'
p = subprocess.Popen(['ant', antcommand], cwd=root_dir,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = p.communicate()
if p.returncode != 0: if p.returncode != 0:
print output raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']))
elif options.verbose: elif options.verbose:
print output print output
print "Build successful"
# Build the release... # Find the apk name in the output...
if thisbuild.has_key('maven'): if thisbuild.has_key('bindir'):
p = subprocess.Popen(['mvn', 'clean', 'install', bindir = os.path.join(build_dir, thisbuild['bindir'])
'-Dandroid.sdk.path=' + sdk_path],
cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
if thisbuild.has_key('antcommand'):
antcommand = thisbuild['antcommand']
else: else:
antcommand = 'release' bindir = os.path.join(root_dir, 'bin')
p = subprocess.Popen(['ant', antcommand], cwd=root_dir, if thisbuild.get('initfun', 'no') == "yes":
stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Special case (again!) for funambol...
output, error = p.communicate() src = ("funambol-android-sync-client-" +
if p.returncode != 0: thisbuild['version'] + "-unsigned.apk")
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) src = os.path.join(bindir, src)
elif options.verbose: elif thisbuild.has_key('maven'):
print output src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
print "Build successful" output, re.S|re.M).group(1)
src = os.path.join(bindir, src) + '.apk'
# Find the apk name in the output...
if thisbuild.has_key('bindir'):
bindir = os.path.join(build_dir, thisbuild['bindir'])
else:
bindir = os.path.join(root_dir, 'bin')
if thisbuild.get('initfun', 'no') == "yes":
# Special case (again!) for funambol...
src = ("funambol-android-sync-client-" +
thisbuild['version'] + "-unsigned.apk")
src = os.path.join(bindir, src)
elif thisbuild.has_key('maven'):
src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
output, re.S|re.M).group(1)
src = os.path.join(bindir, src) + '.apk'
#[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk #[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk
else: else:
src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, src = re.match(r".*^.*Creating (\S+) for release.*$.*", output,
re.S|re.M).group(1) re.S|re.M).group(1)
src = os.path.join(bindir, src) src = os.path.join(bindir, src)
# By way of a sanity check, make sure the version and version # By way of a sanity check, make sure the version and version
# code in our new apk match what we expect... # code in our new apk match what we expect...
print "Checking " + src print "Checking " + src
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
'aapt'), 'aapt'),
'dump', 'badging', src], 'dump', 'badging', src],
stdout=subprocess.PIPE) stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]
if thisbuild.get('novcheck', 'no') == "yes": if thisbuild.get('novcheck', 'no') == "yes":
vercode = thisbuild['vercode'] vercode = thisbuild['vercode']
version = thisbuild['version'] version = thisbuild['version']
else: else:
vercode = None vercode = None
version = None version = None
for line in output.splitlines(): for line in output.splitlines():
if line.startswith("package:"): if line.startswith("package:"):
pat = re.compile(".*versionCode='([0-9]*)'.*") pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1) vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*") pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1) version = re.match(pat, line).group(1)
if version == None or vercode == None: if version == None or vercode == None:
raise BuildException("Could not find version information in build in output") raise BuildException("Could not find version information in build in output")
# Some apps (e.g. Timeriffic) have had the bonkers idea of # Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove # including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it # it so we can compare. (TODO: might be better to remove it
# before we compile, in fact) # before we compile, in fact)
index = version.find(" //") index = version.find(" //")
if index != -1: if index != -1:
version = version[:index] version = version[:index]
if (version != thisbuild['version'] or if (version != thisbuild['version'] or
vercode != thisbuild['vercode']): vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output" raise BuildException(("Unexpected version/version code in output"
"APK: %s / %s" "APK: %s / %s"
"Expected: %s / %s") "Expected: %s / %s")
% (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
) )
# Copy the unsigned apk to our destination directory for further # Copy the unsigned apk to our destination directory for further
# processing (by publish.py)... # processing (by publish.py)...
shutil.copyfile(src, dest) shutil.copyfile(src, dest)
# Move the source tarball into the output directory... # Move the source tarball into the output directory...
if output_dir != tmp_dir: if output_dir != tmp_dir:
tarfilename = tarname + '.tar.gz' tarfilename = tarname + '.tar.gz'
shutil.move(os.path.join(tmp_dir, tarfilename), shutil.move(os.path.join(tmp_dir, tarfilename),
os.path.join(output_dir, tarfilename)) os.path.join(output_dir, tarfilename))
build_succeeded.append(app) build_succeeded.append(app)
except BuildException as be: except BuildException as be:
@ -264,7 +343,7 @@ for app in apps:
except Exception as e: except Exception as e:
if options.stop: if options.stop:
raise raise
print "Could not build app %s due to unknown error: %s" % (app['id'], e) print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
failed_apps[app['id']] = e failed_apps[app['id']] = e
for app in build_succeeded: for app in build_succeeded:

1
builder/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
sshconfig

8
builder/Vagrantfile vendored Normal file
View File

@ -0,0 +1,8 @@
Vagrant::Config.run do |config|
config.vm.box = "buildserver"
config.vm.box_url = "../buildserver.box"
config.vm.customize ["modifyvm", :id, "--memory", "768"]
end

1
buildserver/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.vagrant

26
buildserver/Vagrantfile vendored Normal file
View File

@ -0,0 +1,26 @@
Vagrant::Config.run do |config|
config.vm.box = "debian6-32"
config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box"
config.vm.customize ["modifyvm", :id, "--memory", "1024"]
config.vm.provision :shell, :path => "fixpaths.sh"
# Set apt proxy - remove, or adjust this, accordingly!
config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://thurlow:3142\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
config.vm.provision :chef_solo do |chef|
chef.cookbooks_path = "cookbooks"
chef.log_level = :debug
chef.json = {
:settings => {
:sdk_loc => "/home/vagrant/android-sdk",
:ndk_loc => "/home/vagrant/android-ndk",
:user => "vagrant"
}
}
chef.add_recipe "android-sdk"
chef.add_recipe "android-ndk"
chef.add_recipe "fdroidbuild-general"
end
end

17
buildserver/cookbooks/android-ndk/recipes/default.rb Normal file
View File

@ -0,0 +1,17 @@
ndk_loc = node[:settings][:ndk_loc]
script "setup-android-ndk" do
interpreter "bash"
user node[:settings][:user]
cwd "/tmp"
code "
wget http://dl.google.com/android/ndk/android-ndk-r7-linux-x86.tar.bz2
tar jxvf android-ndk-r7-linux-x86.tar.bz2
mv android-ndk-r7 #{ndk_loc}
"
not_if do
File.exists?("#{ndk_loc}")
end
end

34
buildserver/cookbooks/android-sdk/recipes/default.rb Normal file
View File

@ -0,0 +1,34 @@
%w{openjdk-6-jdk}.each do |pkg|
package pkg do
action :install
end
end
sdk_loc = node[:settings][:sdk_loc]
user = node[:settings][:user]
script "setup-android-sdk" do
interpreter "bash"
user user
cwd "/tmp"
code "
wget http://dl.google.com/android/android-sdk_r16-linux.tgz
tar zxvf android-sdk_r16-linux.tgz
mv android-sdk-linux #{sdk_loc}
rm android-sdk_r16-linux.tgz
#{sdk_loc}/tools/android update sdk --no-ui -t platform-tool
#{sdk_loc}/tools/android update sdk --no-ui -t platform
#{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool
"
not_if do
File.exists?("#{sdk_loc}")
end
end
execute "add-android-sdk-path" do
user user
path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools"
command "echo \"export PATH=\\$PATH:#{path}\" >> /home/#{user}/.bashrc"
not_if "grep #{sdk_loc} /home/#{user}/.bashrc"
end

7
buildserver/cookbooks/fdroidbuild-general/recipes/default.rb Normal file
View File

@ -0,0 +1,7 @@
%w{ant ant-contrib maven2 javacc python}.each do |pkg|
package pkg do
action :install
end
end

23
buildserver/fixpaths.sh Normal file
View File

@ -0,0 +1,23 @@
#!/bin/sh
fixit()
{
#Fix sudoers so the PATH gets passed through, otherwise chef
#provisioning doesn't work.
if [ -z "$1" ]; then
export EDITOR=$0 && sudo -E visudo
else
echo "Fix sudoers"
echo "Defaults exempt_group=admin" >> $1
fi
#Stick the gems bin onto root's path as well.
sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc
# Restart sudo so it gets the changes straight away
sudo /etc/init.d/sudo restart
}
sudo grep "exempt_group" /etc/sudoers -q
if [ "$?" -eq "1" ]; then
fixit
fi

2
common.py
View File

@ -892,7 +892,7 @@ def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, j
# Scan the source code in the given directory (and all subdirectories) # Scan the source code in the given directory (and all subdirectories)
# and return a list of potential problems. # and return a list of potential problems.
def scan_source(source_dir): def scan_source(build_dir, root_dir, thisbuild):
problems = [] problems = []

4
config.buildserver.py Normal file
View File

@ -0,0 +1,4 @@
aapt_path = "/home/vagrant/android-sdk/platform-tools/aapt"
sdk_path = "/home/vagrant/android-sdk"
ndk_path = "/home/vagrant/android-ndk"
javacc_path = "/usr/share/java"

2
scanner.py
View File

@ -93,7 +93,7 @@ for app in apps:
refreshed_source = True refreshed_source = True
# Do the scan... # Do the scan...
buildprobs = common.scan_source(build_dir) buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
for problem in buildprobs: for problem in buildprobs:
problems.append(problem + problems.append(problem +
' in ' + app['id'] + ' ' + thisbuild['version']) ' in ' + app['id'] + ' ' + thisbuild['version'])