diff --git a/buildserver/config.buildserver.py b/buildserver/config.buildserver.py index 3982c2ee..9e511096 100644 --- a/buildserver/config.buildserver.py +++ b/buildserver/config.buildserver.py @@ -1,5 +1,8 @@ sdk_path = "/home/vagrant/android-sdk" -ndk_path = "/home/vagrant/android-ndk" +ndk_paths = { + 'r9b': "/home/vagrant/android-ndk/r9b", + 'r10d': "/home/vagrant/android-ndk/r10d" +} build_tools = "21.1.2" ant = "ant" mvn3 = "mvn" diff --git a/buildserver/cookbooks/android-ndk/recipes/default.rb b/buildserver/cookbooks/android-ndk/recipes/default.rb index 460b4fc4..05b61e89 100644 --- a/buildserver/cookbooks/android-ndk/recipes/default.rb +++ b/buildserver/cookbooks/android-ndk/recipes/default.rb @@ -2,13 +2,7 @@ ndk_loc = node[:settings][:ndk_loc] user = node[:settings][:user] -execute "add-android-ndk-path" do - user user - command "echo \"export PATH=\\$PATH:#{ndk_loc} #PATH-NDK\" >> /home/#{user}/.bsenv" - not_if "grep PATH-NDK /home/#{user}/.bsenv" -end - -script "setup-android-ndk" do +script "setup-android-ndk-r9b" do timeout 14400 interpreter "bash" user node[:settings][:user] @@ -21,10 +15,29 @@ script "setup-android-ndk" do fi tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 - mv android-ndk-r9b #{ndk_loc} + mv android-ndk-r9b #{ndk_loc}/r9b " not_if do - File.exists?("#{ndk_loc}") + File.exists?("#{ndk_loc}/r9b") + end +end + +script "setup-android-ndk-r10d" do + timeout 14400 + interpreter "bash" + user node[:settings][:user] + cwd "/tmp" + code " + if [ `uname -m` == 'x86_64' ] ; then + SUFFIX='_64' + else + SUFFIX='' + fi + /vagrant/cache/android-ndk-r10d-linux-x86$SUFFIX.bin x + mv android-ndk-r10d #{ndk_loc}/r10d + " + not_if do + File.exists?("#{ndk_loc}/r10d") end end diff --git a/docs/fdroid.texi b/docs/fdroid.texi index 3226387d..a1d01077 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -988,6 +988,15 @@ actually not required or used, remove the directory instead (using isn't used nor built will result in an error saying that native libraries were expected in the resulting package. +@item ndk= +Version of the NDK to use in this build. Defaults to the latest NDK release +that included legacy toolchains, so as to not break builds that require +toolchains no longer included in current versions of the NDK. + +The buildserver supports r9b with its legacy toolchains and the latest release +as of writing this document, r10d. You may add support for more versions by +adding them to 'ndk_paths' in your config file. + @item gradle=[,,...] Build with Gradle instead of Ant, specifying what flavours to use. Flavours are case sensitive since the path to the output apk is as well. diff --git a/examples/config.py b/examples/config.py index f1c6f7b9..0f2bf990 100644 --- a/examples/config.py +++ b/examples/config.py @@ -3,24 +3,27 @@ # Copy this file to config.py, then amend the settings below according to # your system configuration. -# Override the path to the Android SDK, $ANDROID_HOME by default -# sdk_path = "/path/to/android-sdk" +# Path to the Android SDK +sdk_path = "$ANDROID_HOME" + +# Path to various versions of the Android NDK +# Most users will have the latest at $ANDROID_NDK, which is used by default +# If a version is missing or assigned to None, it is assumed not installed +ndk_paths = { + 'r9b': None, + 'r10d': "$ANDROID_NDK" +} -# Override the path to the Android NDK, $ANDROID_NDK by default -# ndk_path = "/path/to/android-ndk" # Build tools version to be used -# build_tools = "21.1.2" +build_tools = "21.1.2" -# Command for running Ant -# ant = "/path/to/ant" +# Command or path to binary for running Ant ant = "ant" -# Command for running maven 3 -# mvn3 = "/path/to/mvn" +# Command or path to binary for running maven 3 mvn3 = "mvn" -# Command for running Gradle -# gradle = "/path/to/gradle" +# Command or path to binary for running Gradle gradle = "gradle" # Set the maximum age (in days) of an index that a client should accept from diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 0a503e4b..af8d2ae6 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -452,13 +452,22 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d """Do a build locally.""" if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: - if not config['ndk_path']: - logging.critical("$ANDROID_NDK is not set!") + if not thisbuild['ndk_path']: + logging.critical("Android NDK version '%s' could not be found!" % thisbuild['ndk']) + logging.critical("Configured versions:") + for k, v in config['ndk_paths'].iteritems(): + if k.endswith("_orig"): + continue + logging.critical(" %s: %s" % (k, v)) sys.exit(3) - elif not os.path.isdir(config['sdk_path']): - logging.critical("$ANDROID_NDK points to a non-existing directory!") + elif not os.path.isdir(thisbuild['ndk_path']): + logging.critical("Android NDK '%s' is not a directory!" % thisbuild['ndk_path']) sys.exit(3) + # Set up environment vars that depend on each build + for n in ['ANDROID_NDK', 'NDK']: + common.env[n] = thisbuild['ndk_path'] + # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, @@ -551,7 +560,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d # Run a build command if one is required... if thisbuild['build']: logging.info("Running 'build' commands in %s" % root_dir) - cmd = common.replace_config_vars(thisbuild['build']) + cmd = common.replace_config_vars(thisbuild['build'], thisbuild) # Substitute source library paths into commands... for name, number, libpath in srclibpaths: @@ -571,7 +580,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if jni_components == ['yes']: jni_components = [''] - cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"] + cmd = [os.path.join(thisbuild['ndk_path'], "ndk-build"), "-j1"] for d in jni_components: if d: logging.info("Building native code in '%s'" % d) @@ -644,8 +653,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d modules = bconfig.get('app', 'requirements').split(',') cmd = 'ANDROIDSDK=' + config['sdk_path'] - cmd += ' ANDROIDNDK=' + config['ndk_path'] - cmd += ' ANDROIDNDKVER=r9' + cmd += ' ANDROIDNDK=' + thisbuild['ndk_path'] + cmd += ' ANDROIDNDKVER=' + thisbuild['ndk'] cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api')) cmd += ' VIRTUALENV=virtualenv' cmd += ' ./distribute.sh' @@ -1042,7 +1051,7 @@ def main(): # Set up vcs interface and make sure we have the latest code... logging.debug("Getting {0} vcs interface for {1}" .format(app['Repo Type'], app['Repo'])) - vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, build) first = False diff --git a/fdroidserver/common.py b/fdroidserver/common.py index b9e7b81c..db65bcb7 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -41,7 +41,10 @@ env = None default_config = { 'sdk_path': "$ANDROID_HOME", - 'ndk_path': "$ANDROID_NDK", + 'ndk_paths': { + 'r9b': None, + 'r10d': "$ANDROID_NDK" + }, 'build_tools': "21.1.2", 'ant': "ant", 'mvn3': "mvn", @@ -82,14 +85,31 @@ def fill_config_defaults(thisconfig): thisconfig[k] = v # Expand paths (~users and $vars) - for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']: + def expand_path(path): + if path is None: + return None + orig = path + path = os.path.expanduser(path) + path = os.path.expandvars(path) + if orig == path: + return None + return path + + for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']: v = thisconfig[k] - orig = v - v = os.path.expanduser(v) - v = os.path.expandvars(v) - if orig != v: - thisconfig[k] = v - thisconfig[k + '_orig'] = orig + exp = expand_path(v) + if exp is not None: + thisconfig[k] = exp + thisconfig[k + '_orig'] = v + + for k in ['ndk_paths']: + d = thisconfig[k] + for k2 in d.copy(): + v = d[k2] + exp = expand_path(v) + if exp is not None: + thisconfig[k][k2] = exp + thisconfig[k][k2 + '_orig'] = v def read_config(opts, config_file='config.py'): @@ -135,8 +155,6 @@ def read_config(opts, config_file='config.py'): env = os.environ for n in ['ANDROID_HOME', 'ANDROID_SDK']: env[n] = config['sdk_path'] - for n in ['ANDROID_NDK', 'NDK']: - env[n] = config['ndk_path'] for k in ["keystorepass", "keypass"]: if k in config: @@ -165,6 +183,15 @@ def read_config(opts, config_file='config.py'): return config +def get_ndk_path(version): + if version is None: + version = 'r10d' # latest + paths = config['ndk_paths'] + if version not in paths: + return None + return paths[version] + + def find_sdk_tools_cmd(cmd): '''find a working path to a tool from the Android SDK''' @@ -366,7 +393,7 @@ def getcvname(app): return '%s (%s)' % (app['Current Version'], app['Current Version Code']) -def getvcs(vcstype, remote, local): +def getvcs(vcstype, remote, local, build): if vcstype == 'git': return vcs_git(remote, local) if vcstype == 'git-svn': @@ -378,7 +405,7 @@ def getvcs(vcstype, remote, local): if vcstype == 'srclib': if local != os.path.join('build', 'srclib', remote): raise VCSException("Error: srclib paths are hard-coded!") - return getsrclib(remote, os.path.join('build', 'srclib'), raw=True) + return getsrclib(remote, os.path.join('build', 'srclib'), build, raw=True) if vcstype == 'svn': raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead") raise VCSException("Invalid vcs type " + vcstype) @@ -1040,7 +1067,7 @@ class BuildException(FDroidException): # Returns the path to it. Normally this is the path to be used when referencing # it, which may be a subdirectory of the actual project. If you want the base # directory of the project, pass 'basepath=True'. -def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, +def getsrclib(spec, srclib_dir, build, srclibpaths=[], subdir=None, basepath=False, raw=False, prepare=True, preponly=False): number = None @@ -1063,7 +1090,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, sdir = os.path.join(srclib_dir, name) if not preponly: - vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir) + vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir, build) vcs.srclib = (name, number, sdir) if ref: vcs.gotorevision(ref) @@ -1104,7 +1131,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, if prepare: if srclib["Prepare"]: - cmd = replace_config_vars(srclib["Prepare"]) + cmd = replace_config_vars(srclib["Prepare"], build) p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir) if p.returncode != 0: @@ -1155,7 +1182,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Run an init command if one is required if build['init']: - cmd = replace_config_vars(build['init']) + cmd = replace_config_vars(build['init'], build) logging.info("Running 'init' commands in %s" % root_dir) p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) @@ -1179,7 +1206,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if build['srclibs']: logging.info("Collecting source libraries") for lib in build['srclibs']: - srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths, + srclibpaths.append(getsrclib(lib, srclib_dir, build, srclibpaths, preponly=onserver)) for name, number, libpath in srclibpaths: @@ -1213,10 +1240,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= else: props += "sdk.dir=%s\n" % config['sdk_path'] props += "sdk-location=%s\n" % config['sdk_path'] - if config['ndk_path']: + if build['ndk_path']: # Add ndk location - props += "ndk.dir=%s\n" % config['ndk_path'] - props += "ndk-location=%s\n" % config['ndk_path'] + props += "ndk.dir=%s\n" % build['ndk_path'] + props += "ndk-location=%s\n" % build['ndk_path'] # Add java.encoding if necessary if build['encoding']: props += "java.encoding=%s\n" % build['encoding'] @@ -1336,7 +1363,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if build['prebuild']: logging.info("Running 'prebuild' commands in %s" % root_dir) - cmd = replace_config_vars(build['prebuild']) + cmd = replace_config_vars(build['prebuild'], build) # Substitute source library paths into prebuild commands for name, number, libpath in srclibpaths: @@ -1764,9 +1791,9 @@ def remove_signing_keys(build_dir): logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path)) -def replace_config_vars(cmd): +def replace_config_vars(cmd, build): cmd = cmd.replace('$$SDK$$', config['sdk_path']) - cmd = cmd.replace('$$NDK$$', config['ndk_path']) + cmd = cmd.replace('$$NDK$$', build['ndk_path']) cmd = cmd.replace('$$MVN3$$', config['mvn3']) return cmd diff --git a/fdroidserver/init.py b/fdroidserver/init.py index be62f103..2e16efbd 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -203,16 +203,8 @@ def main(): # now that we have a local config.py, read configuration... config = common.read_config(options) - # track down where the Android NDK is - ndk_path = '/opt/android-ndk' - if os.path.isdir(config['ndk_path']): - ndk_path = config['ndk_path'] - elif 'ANDROID_NDK' in os.environ.keys(): - logging.info('using ANDROID_NDK') - ndk_path = os.environ['ANDROID_NDK'] - if os.path.isdir(ndk_path): - write_to_config(test_config, 'ndk_path') - # the NDK is optional so we don't prompt the user for it if its not found + # the NDK is optional and there may be multiple versions of it, so it's + # left for the user to configure # find or generate the keystore for the repo signing key. First try the # path written in the default config.py. Then check if the user has @@ -286,7 +278,7 @@ def main(): logging.info(' Android SDK:\t\t\t' + config['sdk_path']) if aapt: logging.info(' Android SDK Build Tools:\t' + os.path.dirname(aapt)) - logging.info(' Android NDK (optional):\t' + ndk_path) + logging.info(' Android NDK r10d (optional):\t$ANDROID_NDK') logging.info(' Keystore for signing key:\t' + keystore) if repo_keyalias is not None: logging.info(' Alias for key in store:\t' + repo_keyalias) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 1ba5a713..11017122 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -25,6 +25,8 @@ import logging from collections import OrderedDict +import common + srclibs = None @@ -100,6 +102,7 @@ flag_defaults = OrderedDict([ ('scandelete', []), ('build', ''), ('buildjni', []), + ('ndk', 'r9b'), # defaults to oldest ('preassemble', []), ('antcommands', None), ('novcheck', False), @@ -560,6 +563,7 @@ def fill_build_defaults(build): continue build[flag] = value build['type'] = get_build_type() + build['ndk_path'] = common.get_ndk_path(build['ndk']) # Parse metadata for a single application. diff --git a/makebuildserver b/makebuildserver index 6dc317dc..54c7eed8 100755 --- a/makebuildserver +++ b/makebuildserver @@ -102,6 +102,9 @@ cachefiles = [ if config['arch64']: cachefiles.extend([ + ('android-ndk-r10d-linux-x86_64.bin', + 'https://dl.google.com/android/ndk/android-ndk-r10d-linux-x86_64.bin', + '812949f9299afd4b91890863054dc42f6547b6d485211d5f0faca9f286685df6'), ('android-ndk-r9b-linux-x86_64.tar.bz2', 'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2', '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'), @@ -110,6 +113,9 @@ if config['arch64']: 'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1')]) else: cachefiles.extend([ + ('android-ndk-r10d-linux-x86.bin', + 'https://dl.google.com/android/ndk/android-ndk-r10d-linux-x86.bin', + 'c0d07e5ce2fff13b5eb456c10e99527184c9139e798cb7fd1adfadafa65cb696'), ('android-ndk-r9b-linux-x86.tar.bz2', 'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86.tar.bz2', '748104b829dd12afb2fdb3044634963abb24cdb0aad3b26030abe2e9e65bfc81'),