kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-19 03:30:12 +02:00
Code Activity

Merge branch 'master' of git://gitorious.org/f-droid/fdroidserver

Browse Source
This commit is contained in:
Dominik 2012-02-21 18:48:14 +01:00
commit 940c85e4da
9 changed files with 251 additions and 221 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
README.buildserver
View File

@ -1,15 +1,14 @@
Integrating the build server setup into the main scripts is a work in progress. Some things may
not work properly yet. Talk to CiaranG if you're trying to use this and have problems.
Setting up a build server:
1. Install VirtualBox, vagrant and vagrant-snap
2. In the buildserver directory, run 'vagrant up'. Will take a long time.
3. Log in with 'vagrant ssh'
4. Check it all looks ok, then 'sudo shutdown -h now'
5. Back in the main directory, run 'VBoxManage listvms' look for
buildserver_xxxx
6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'.
Will take a while.
7. You should now have a new 'buildserver.box'
2. Create (or get - ask CiaranG, or wait until I replace this with a download link!) a standard
Debian Squeeze vagrant-compatible base box called 'debian6-32'
3. Run makebuildserver.sh. This will take a long time. The end result is a new base box called
'buildserver'.
You should now be able to use the --server option on build.py and builds will
take place in the clean, secure, isolated environment of a fresh virtual

254
build.py
View File

@ -32,110 +32,18 @@ import common
from common import BuildException
from common import VCSException
#Read configuration...
execfile('config.py')
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Build only the specified package")
parser.add_option("-s", "--stop", action="store_true", default=False,
help="Make the build stop on exceptions")
parser.add_option("-t", "--test", action="store_true", default=False,
help="Test mode - put output in the tmp directory only.")
parser.add_option("--server", action="store_true", default=False,
help="Use build server")
parser.add_option("--on-server", action="store_true", default=False,
help="Specify that we're running on the build server")
parser.add_option("-f", "--force", action="store_true", default=False,
help="Force build of disabled app. Only allowed in test mode.")
(options, args) = parser.parse_args()
if options.force and not options.test:
print "Force is only allowed in test mode"
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose)
failed_apps = {}
build_succeeded = []
log_dir = 'logs'
if not os.path.isdir(log_dir):
print "Creating log directory"
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
print "Creating output directory"
os.makedirs(output_dir)
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
extlib_dir = os.path.join(build_dir, 'extlib')
# Build applications...
for app in apps:
if options.package and options.package != app['id']:
# Silent skip...
pass
elif app['Disabled'] and not options.force:
if options.verbose:
print "Skipping %s: disabled" % app['id']
elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
if options.verbose:
print "Skipping %s: no builds specified" % app['id']
else:
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
refreshed_source = False
for thisbuild in app['builds']:
try:
dest = os.path.join(output_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk')
dest_repo = os.path.join(repo_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk')
if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
if options.verbose:
print "..version " + thisbuild['version'] + " already exists"
elif thisbuild['commit'].startswith('!'):
if options.verbose:
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild['commit'][1:])
else:
if options.verbose:
mstart = '.. building version '
else:
mstart = 'Building version '
print mstart + thisbuild['version'] + ' of ' + app['id']
if options.server:
# Do a build on the build server.
def build_server(app, thisbuild, build_dir, output_dir):
import paramiko
# Destroy the builder vm if it already exists...
# TODO: need to integrate the snapshot stuff so it doesn't have to
# keep wasting time doing this unnecessarily.
if os.path.exists(os.path.join('builder', '.vagrant')):
if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0:
raise BuildException("Failed to destroy build server")
# Start up the virtual maachine...
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
# Not a very helpful message yet!
@ -169,30 +77,37 @@ for app in apps:
ftp.put('build.py', 'build.py')
ftp.put('common.py', 'common.py')
ftp.put('config.buildserver.py', 'config.py')
ftp.mkdir('metadata')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
app['id'] + '.txt')
ftp.chdir('..')
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir(app['id'])
ftp.chdir('..')
def send_dir(path):
lastdir = path
for r, d, f in os.walk(base):
while lastdir != os.path.commonprefix([lastdir, root]):
ftp.chdir('..')
lastdir = os.path.split(lastdir)[0]
lastdir = r
for r, d, f in os.walk(path):
ftp.chdir(r)
for dd in d:
ftp.mkdir(dd)
for ff in f:
ftp.put(os.path.join(r, ff), ff)
send_dir(app['id'])
# TODO: send relevant extlib directories too
ftp.chdir('/home/vagrant')
for i in range(len(r.split('/'))):
ftp.chdir('..')
send_dir(build_dir)
# TODO: send relevant extlib and srclib directories too
# Execute the build script...
ssh.exec_command('python build.py --on-server -p ' +
app['id'])
app['id'] + ' --vercode ' + thisbuild['vercode'])
# Retrieve the built files...
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
ftp.chdir('unsigned')
ftp.chdir('/home/vagrant/unsigned')
ftp.get(apkfile, os.path.join(output_dir, apkfile))
ftp.get(tarball, os.path.join(output_dir, tarball))
@ -201,13 +116,14 @@ for app in apps:
# Not a very helpful message yet!
raise BuildException("Failed to destroy")
else:
# Do a build locally.
def build_local(app, thisbuild, build_dir, output_dir):
# Prepare the source code...
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path,
javacc_path, not refreshed_source)
refreshed_source = True
javacc_path)
# Scan before building...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
@ -329,6 +245,116 @@ for app in apps:
shutil.move(os.path.join(tmp_dir, tarfilename),
os.path.join(output_dir, tarfilename))
#Read configuration...
execfile('config.py')
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Build only the specified package")
parser.add_option("-c", "--vercode", default=None,
help="Build only the specified version code")
parser.add_option("-s", "--stop", action="store_true", default=False,
help="Make the build stop on exceptions")
parser.add_option("-t", "--test", action="store_true", default=False,
help="Test mode - put output in the tmp directory only.")
parser.add_option("--server", action="store_true", default=False,
help="Use build server")
parser.add_option("--on-server", action="store_true", default=False,
help="Specify that we're running on the build server")
parser.add_option("-f", "--force", action="store_true", default=False,
help="Force build of disabled app. Only allowed in test mode.")
(options, args) = parser.parse_args()
if options.force and not options.test:
print "Force is only allowed in test mode"
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose)
failed_apps = {}
build_succeeded = []
log_dir = 'logs'
if not os.path.isdir(log_dir):
print "Creating log directory"
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
print "Creating output directory"
os.makedirs(output_dir)
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
extlib_dir = os.path.join(build_dir, 'extlib')
# Filter apps and build versions according to command-line options...
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if options.vercode:
for app in apps:
app['builds'] = [b for b in app['builds']
if str(b['vercode']) == options.vercode]
# Build applications...
for app in apps:
if app['Disabled'] and not options.force:
if options.verbose:
print "Skipping %s: disabled" % app['id']
elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0:
if options.verbose:
print "Skipping %s: no builds specified" % app['id']
else:
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
for thisbuild in app['builds']:
try:
dest = os.path.join(output_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk')
dest_repo = os.path.join(repo_dir, app['id'] + '_' +
thisbuild['vercode'] + '.apk')
if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)):
if options.verbose:
print "..version " + thisbuild['version'] + " already exists"
elif thisbuild['commit'].startswith('!'):
if options.verbose:
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild['commit'][1:])
else:
if options.verbose:
mstart = '.. building version '
else:
mstart = 'Building version '
print mstart + thisbuild['version'] + ' of ' + app['id']
if options.server:
build_server(app, thisbuild, build_dir, output_dir)
else:
build_local(app, thisbuild, build_dir, output_dir)
build_succeeded.append(app)
except BuildException as be:
if options.stop:

1
builder/.gitignore vendored
View File

@ -1,2 +1,3 @@
sshconfig
.vagrant
*.log

1
builder/Vagrantfile vendored
View File

@ -1,7 +1,6 @@
Vagrant::Config.run do |config|
config.vm.box = "buildserver"
config.vm.box_url = "../buildserver.box"
config.vm.customize ["modifyvm", :id, "--memory", "768"]

2
buildserver/cookbooks/fdroidbuild-general/recipes/default.rb
View File

@ -1,5 +1,5 @@
%w{ant ant-contrib maven2 javacc python}.each do |pkg|
%w{ant ant-contrib maven2 javacc python git-core mercurial subversion bzr}.each do |pkg|
package pkg do
action :install
end

3
common.py
View File

@ -657,10 +657,9 @@ def getsrclib(spec, extlib_dir):
# 'sdk_path' - the path to the Android SDK
# 'ndk_path' - the path to the Android NDK
# 'javacc_path' - the path to javacc
# 'refresh' - True to refresh from the remote repo
# Returns the root directory, which may be the same as 'build_dir' or may
# be a subdirectory of it.
def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, refresh):
def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path):
# Optionally, the actual app source can be in a subdirectory...
if build.has_key('subdir'):

11
makebuildserver.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
rm -f buildserver.box
cd buildserver
vagrant up
vagrant ssh -c "sudo shutdown -h now"
cd ..
# Just to wait until it's shut down!
sleep 20
vagrant package --base `VBoxManage list vms | grep buildserver | sed 's/"\(.*\)".*/\1/'` --output buildserver.box
vagrant box add buildserver buildserver.box -f && rm buildserver.box

7
scanner.py
View File

@ -75,9 +75,6 @@ for app in apps:
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
refreshed_source = False
for thisbuild in app['builds']:
if thisbuild['commit'].startswith('!'):
@ -88,9 +85,7 @@ for app in apps:
# Prepare the source code...
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path, javacc_path,
not refreshed_source)
refreshed_source = True
build_dir, extlib_dir, sdk_path, ndk_path, javacc_path)
# Do the scan...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)

4
wp-fdroid/wp-fdroid.php
View File

@ -253,7 +253,7 @@ class FDroid
foreach($apks as $apk) {
$first = $i+1==count($apks);
$out.="<p><b>Version ".$apk['version']."</b><br />";
$out.='<a href="http://f-droid.org/repo/'.$apk['apkname'].'">download apk</a> ';
$out.='<a href="https://f-droid.org/repo/'.$apk['apkname'].'">download apk</a> ';
$out.=$this->human_readable_size($apk['size']);
$diffSize = $apk['diff']['size'];
if(abs($diffSize) > 500) {
@ -262,7 +262,7 @@ class FDroid
$out.=$this->human_readable_size($diffSize, 1).')</span>';
}
if(isset($apk['srcname']) && file_exists($this->site_path.'/repo/'.$apk['srcname'])) {
$out.='<br /><a href="http://f-droid.org/repo/'.$apk['srcname'].'">source tarball</a> ';
$out.='<br /><a href="https://f-droid.org/repo/'.$apk['srcname'].'">source tarball</a> ';
$out.=$this->human_readable_size(filesize($this->site_path.'/repo/'.$apk['srcname']));
}