From 09e296d2de1f53259bc6133bb6939f86646d880b Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Sat, 8 Sep 2012 09:56:20 +0100 Subject: [PATCH] Lots of build system improvements and fixes --- .../cookbooks/android-sdk/recipes/default.rb | 2 +- fdroidserver/build.py | 38 +++++------ fdroidserver/checkupdates.py | 12 ++-- fdroidserver/common.py | 67 +++++++++++-------- fdroidserver/import.py | 2 +- fdroidserver/scanner.py | 2 +- 6 files changed, 66 insertions(+), 57 deletions(-) diff --git a/buildserver/cookbooks/android-sdk/recipes/default.rb b/buildserver/cookbooks/android-sdk/recipes/default.rb index b16aba97..4e6e976f 100644 --- a/buildserver/cookbooks/android-sdk/recipes/default.rb +++ b/buildserver/cookbooks/android-sdk/recipes/default.rb @@ -44,7 +44,7 @@ end end -%w{addon-google_apis-google-7 addon-google_apis-google-16}.each do |sdk| +%w{addon-google_apis-google-7 addon-google_apis-google-15 addon-google_apis-google-16}.each do |sdk| script "add_addon_#{sdk}" do interpreter "bash" diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 4efb4aa3..7ebae359 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -33,7 +33,7 @@ from common import BuildException from common import VCSException -def build_server(app, thisbuild, build_dir, output_dir): +def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path): """Do a build on the build server.""" import ssh @@ -152,7 +152,10 @@ def build_server(app, thisbuild, build_dir, output_dir): ftp.chdir('metadata') ftp.put(os.path.join('metadata', app['id'] + '.txt'), app['id'] + '.txt') - ftp.chdir('..') + # And patches if there are any... + if os.path.exists(os.path.join('metadata', app['id'])): + send_dir(os.path.join('metadata', app['id'])) + ftp.chdir('/home/vagrant') # Create the build directory... ftp.mkdir('build') ftp.chdir('build') @@ -162,8 +165,7 @@ def build_server(app, thisbuild, build_dir, output_dir): send_dir(build_dir) # Copy any extlibs that are required... if thisbuild.has_key('extlibs'): - ftp.chdir('build') - ftp.chdir('extlib') + ftp.chdir('/home/vagrant/build/extlib') for lib in thisbuild['extlibs'].split(';'): lp = lib.split('/') for d in lp[:-1]: @@ -172,22 +174,20 @@ def build_server(app, thisbuild, build_dir, output_dir): ftp.put(os.path.join('build/extlib', lib), lp[-1]) for _ in lp[:-1]: ftp.chdir('..') - ftp.chdir('..') - ftp.chdir('..') # Copy any srclibs that are required... + srclibpaths = [] if thisbuild.has_key('srclibs'): - ftp.chdir('build') - ftp.chdir('extlib') for lib in thisbuild['srclibs'].split(';'): - lp = lib.split('@').split('/') - for d in lp[:-1]: - ftp.mkdir(d) - ftp.chdir(d) - ftp.put(os.path.join('build/extlib', lib), lp[-1]) - for _ in lp[:-1]: - ftp.chdir('..') - ftp.chdir('..') - ftp.chdir('..') + name, _ = lib.split('@') + srclibpaths.append((name, common.getsrclib(lib, 'build/extlib', sdk_path))) + # If one was used for the main source, add that too. + basesrclib = vcs.getsrclib() + if basesrclib: + srclibpaths.append(basesrclib) + print "Sending srclibs:" + for _, lib in srclibpaths: + ftp.chdir('/home/vagrant/build/extlib') + send_dir(lib) # Execute the build script... print "Starting build..." @@ -409,7 +409,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, extlib_dir, tmp_dir, # grabbing the source now. vcs.gotorevision(thisbuild['commit']) - build_server(app, thisbuild, build_dir, output_dir) + build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path) else: build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force, verbose) return True @@ -534,7 +534,7 @@ def main(): build_dir = 'build/' + app['id'] # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path) for thisbuild in app['builds']: try: diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 3d2f7826..1b99b085 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -38,7 +38,7 @@ from common import VCSException # caution, because it's inappropriate for many projects. # Returns (None, "a message") if this didn't work, or (version, vercode) for # the details of the current version. -def check_tags(app): +def check_tags(app, sdk_path): try: @@ -48,7 +48,7 @@ def check_tags(app): return (None, 'Tags update mode only works for git repositories currently') # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path) vcs.gotorevision('origin/master') if len(app['builds']) == 0: @@ -91,7 +91,7 @@ def check_tags(app): # caution, because it's inappropriate for many projects. # Returns (None, "a message") if this didn't work, or (version, vercode) for # the details of the current version. -def check_repomanifest(app): +def check_repomanifest(app, sdk_path): try: @@ -101,7 +101,7 @@ def check_repomanifest(app): return (None, 'RepoManifest update mode only works for git repositories currently') # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path) vcs.gotorevision('origin/master') if len(app['builds']) == 0: @@ -201,9 +201,9 @@ def main(): if mode == 'Market': (version, vercode) = check_market(app) elif mode == 'Tags': - (version, vercode) = check_tags(app) + (version, vercode) = check_tags(app, sdk_path) elif mode == 'RepoManifest': - (version, vercode) = check_repomanifest(app) + (version, vercode) = check_repomanifest(app, sdk_path) elif mode == 'None': version = None vercode = 'Checking disabled' diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 8123b193..49a8cd2f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -22,23 +22,25 @@ import subprocess import time import operator -def getvcs(vcstype, remote, local): +def getvcs(vcstype, remote, local, sdk_path): if vcstype == 'git': - return vcs_git(remote, local) + return vcs_git(remote, local, sdk_path) elif vcstype == 'svn': - return vcs_svn(remote, local) + return vcs_svn(remote, local, sdk_path) elif vcstype == 'git-svn': - return vcs_gitsvn(remote, local) + return vcs_gitsvn(remote, local, sdk_path) elif vcstype == 'hg': - return vcs_hg(remote, local) + return vcs_hg(remote, local, sdk_path) elif vcstype == 'bzr': - return vcs_bzr(remote, local) + return vcs_bzr(remote, local, sdk_path) elif vcstype == 'srclib': - return vcs_srclib(remote, local) + return vcs_srclib(remote, local, sdk_path) raise VCSException("Invalid vcs type " + vcstype) class vcs: - def __init__(self, remote, local): + def __init__(self, remote, local, sdk_path): + + self.sdk_path = sdk_path # It's possible to sneak a username and password in with # the remote address... (this really only applies to svn @@ -329,7 +331,7 @@ class vcs_srclib(vcs): else: srclib = self.remote path = None - libdir = getsrclib(srclib + '@' + rev, extlib_dir) + libdir = getsrclib(srclib + '@' + rev, extlib_dir, self.sdk_path) self.srclib = (srclib, libdir) if path: libdir = os.path.join(libdir, path) @@ -685,23 +687,24 @@ class MetaDataException(Exception): # Returns the path to it. # TODO: These are currently just hard-coded in this method. It will be a # metadata-driven system eventually, but not yet. -def getsrclib(spec, extlib_dir): +def getsrclib(spec, extlib_dir, sdk_path): name, ref = spec.split('@') if name == 'GreenDroid': sdir = os.path.join(extlib_dir, 'GreenDroid') vcs = getvcs('git', - 'https://github.com/cyrilmottier/GreenDroid.git', sdir) + 'https://github.com/cyrilmottier/GreenDroid.git', sdir, sdk_path) vcs.gotorevision(ref) return os.path.join(sdir, 'GreenDroid') if name == 'ActionBarSherlock': sdir = os.path.join(extlib_dir, 'ActionBarSherlock') vcs = getvcs('git', - 'https://github.com/JakeWharton/ActionBarSherlock.git', sdir) + 'https://github.com/JakeWharton/ActionBarSherlock.git', sdir, sdk_path) vcs.gotorevision(ref) libdir = os.path.join(sdir, 'library') - if subprocess.call(['android', 'update', 'project', '-p', + if subprocess.call([os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', libdir]) != 0: raise BuildException('Error updating ActionBarSherlock project') return libdir @@ -709,9 +712,10 @@ def getsrclib(spec, extlib_dir): if name == 'ViewPagerIndicator': sdir = os.path.join(extlib_dir, 'ViewPagerIndicator') vcs = getvcs('git', - 'https://github.com/mariotaku/viewpagerindicator.git', sdir) + 'https://github.com/mariotaku/viewpagerindicator.git', sdir, sdk_path) vcs.gotorevision(ref) - if subprocess.call(['android', 'update', 'project', '-p', + if subprocess.call([os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', sdir]) != 0: raise BuildException('Error updating ViewPagerIndicator project') return sdir @@ -719,10 +723,11 @@ def getsrclib(spec, extlib_dir): if name == 'UITableView': sdir = os.path.join(extlib_dir, 'UITableView') vcs = getvcs('git', - 'https://github.com/thiagolocatelli/android-uitableview.git', sdir) + 'https://github.com/thiagolocatelli/android-uitableview.git', sdir, sdk_path) vcs.gotorevision(ref) libdir = os.path.join(sdir, 'android-uitableview') - if subprocess.call(['android', 'update', 'project', '-p', + if subprocess.call([os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', libdir]) != 0: raise BuildException('Error updating UITableView project') return libdir @@ -730,13 +735,14 @@ def getsrclib(spec, extlib_dir): if name == 'ViewPagerTabs': sdir = os.path.join(extlib_dir, 'ViewPagerTabs') vcs = getvcs('git', - 'https://github.com/astuetz/android-viewpagertabs.git', sdir) + 'https://github.com/astuetz/android-viewpagertabs.git', sdir, sdk_path) vcs.gotorevision(ref) pp = open(os.path.join(sdir, 'project.properties'), 'w') pp.write('android.library=true\n') pp.write('target=android-15\n') pp.close() - if subprocess.call(['android', 'update', 'project', '-p', + if subprocess.call([os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', sdir]) != 0: raise BuildException('Error updating ViewPagerTabs project') return sdir @@ -744,10 +750,11 @@ def getsrclib(spec, extlib_dir): if name == 'ActionBar': sdir = os.path.join(extlib_dir, 'ActionBar') vcs = getvcs('git', - 'https://github.com/johannilsson/android-actionbar.git', sdir) + 'https://github.com/johannilsson/android-actionbar.git', sdir, sdk_path) vcs.gotorevision(ref) libdir = os.path.join(sdir, 'actionbar') - if subprocess.call(['android', 'update', 'project', '-p', + if subprocess.call([os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', libdir]) != 0: raise BuildException('Error updating ActionBar project') return libdir @@ -755,10 +762,11 @@ def getsrclib(spec, extlib_dir): if name == 'ActionBarNW': sdir = os.path.join(extlib_dir, 'ActionBarNW') vcs = getvcs('git', - 'https://github.com/NightWhistler/android-actionbar.git', sdir) + 'https://github.com/NightWhistler/android-actionbar.git', sdir, sdk_path) vcs.gotorevision(ref) libdir = os.path.join(sdir, 'actionbar') - if subprocess.call(['android', 'update', 'project', '-p', + if subprocess.call([os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', libdir]) != 0: raise BuildException('Error updating ActionBarNW project') return libdir @@ -766,10 +774,11 @@ def getsrclib(spec, extlib_dir): if name == 'FacebookSDK': sdir = os.path.join(extlib_dir, 'FacebookSDK') vcs = getvcs('git', - 'git://github.com/facebook/facebook-android-sdk.git', sdir) + 'git://github.com/facebook/facebook-android-sdk.git', sdir, sdk_path) vcs.gotorevision(ref) libdir = os.path.join(sdir, 'facebook') - if subprocess.call(['android', 'update', 'project', '-p', + if subprocess.call([os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', libdir]) != 0: raise BuildException('Error updating FacebookSDK project') return libdir @@ -777,14 +786,14 @@ def getsrclib(spec, extlib_dir): if name == 'OI': sdir = os.path.join(extlib_dir, 'OI') vcs = getvcs('git-svn', - 'http://openintents.googlecode.com/svn/trunk/', sdir) + 'http://openintents.googlecode.com/svn/trunk/', sdir, sdk_path) vcs.gotorevision(ref) return sdir if name == 'JOpenDocument': sdir = os.path.join(extlib_dir, 'JOpenDocument') vcs = getvcs('git', - 'https://github.com/andiwand/JOpenDocument.git', sdir) + 'https://github.com/andiwand/JOpenDocument.git', sdir, sdk_path) vcs.gotorevision(ref) shutil.rmtree(os.path.join(sdir, 'bin')) return sdir @@ -792,7 +801,7 @@ def getsrclib(spec, extlib_dir): if name == 'BitcoinJWallet': sdir = os.path.join(extlib_dir, 'BitcoinJWallet') vcs = getvcs('git', - 'https://code.google.com/r/andreasschildbach-bitcoinj/', sdir) + 'https://code.google.com/r/andreasschildbach-bitcoinj/', sdir, sdk_path) vcs.gotorevision(ref) if subprocess.call(['mvn3', 'install'], cwd=sdir) != 0: raise BuildException("Maven build failed for BitcoinJWallet srclib") @@ -990,7 +999,7 @@ def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, j if build.has_key('srclibs'): for lib in build['srclibs'].split(';'): name, _ = lib.split('@') - srclibpaths.append((name, getsrclib(lib, extlib_dir))) + srclibpaths.append((name, getsrclib(lib, extlib_dir, sdk_path))) basesrclib = vcs.getsrclib() # If one was used for the main source, add that too. if basesrclib: diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 6f435aec..947a6b20 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -212,7 +212,7 @@ def main(): src_dir = os.path.join(tmp_dir, 'importer') if os.path.exists(src_dir): shutil.rmtree(src_dir) - vcs = common.getvcs(repotype, repo, src_dir) + vcs = common.getvcs(repotype, repo, src_dir, sdk_path) vcs.gotorevision(None) if options.subdir: root_dir = os.path.join(src_dir, options.subdir) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 2058b01e..0deccf91 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -84,7 +84,7 @@ def main(): build_dir = 'build/' + app['id'] # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path) for thisbuild in app['builds']: