Add NDK r10d, this time alongside r9b

Specified per-build with ndk=<version> defaulting to the oldest, r9b.

buildserver/config.buildserver.py

@@ -1,5 +1,8 @@
 sdk_path = "/home/vagrant/android-sdk"
-ndk_path = "/home/vagrant/android-ndk"
+ndk_paths = {
+    'r9b': "/home/vagrant/android-ndk/r9b",
+    'r10d': "/home/vagrant/android-ndk/r10d"
+}
 build_tools = "21.1.2"
 ant = "ant"
 mvn3 = "mvn"

buildserver/cookbooks/android-ndk/recipes/default.rb

@@ -2,13 +2,7 @@
 ndk_loc = node[:settings][:ndk_loc]
 user = node[:settings][:user]
 
-execute "add-android-ndk-path" do
+script "setup-android-ndk-r9b" do
-  user user
-  command "echo \"export PATH=\\$PATH:#{ndk_loc} #PATH-NDK\" >> /home/#{user}/.bsenv"
-  not_if "grep PATH-NDK /home/#{user}/.bsenv"
-end
-
-script "setup-android-ndk" do
   timeout 14400
   interpreter "bash"
   user node[:settings][:user]
@@ -21,10 +15,29 @@ script "setup-android-ndk" do
     fi
     tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
     tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
-    mv android-ndk-r9b #{ndk_loc}
+    mv android-ndk-r9b #{ndk_loc}/r9b
   "
   not_if do
-    File.exists?("#{ndk_loc}")
+    File.exists?("#{ndk_loc}/r9b")
+  end
+end
+
+script "setup-android-ndk-r10d" do
+  timeout 14400
+  interpreter "bash"
+  user node[:settings][:user]
+  cwd "/tmp"
+  code "
+    if [ `uname -m` == 'x86_64' ] ; then
+       SUFFIX='_64'
+    else
+       SUFFIX=''
+    fi
+    /vagrant/cache/android-ndk-r10d-linux-x86$SUFFIX.bin x
+    mv android-ndk-r10d #{ndk_loc}/r10d
+  "
+  not_if do
+    File.exists?("#{ndk_loc}/r10d")
   end
 end
 

docs/fdroid.texi

@@ -988,6 +988,15 @@ actually not required or used, remove the directory instead (using
 isn't used nor built will result in an error saying that native
 libraries were expected in the resulting package.
 
+@item ndk=<version>
+Version of the NDK to use in this build. Defaults to the latest NDK release
+that included legacy toolchains, so as to not break builds that require
+toolchains no longer included in current versions of the NDK.
+
+The buildserver supports r9b with its legacy toolchains and the latest release
+as of writing this document, r10d. You may add support for more versions by
+adding them to 'ndk_paths' in your config file.
+
 @item gradle=<flavour1>[,<flavour2>,...]
 Build with Gradle instead of Ant, specifying what flavours to use. Flavours
 are case sensitive since the path to the output apk is as well.

examples/config.py

@@ -3,24 +3,27 @@
 # Copy this file to config.py, then amend the settings below according to
 # your system configuration.
 
-# Override the path to the Android SDK, $ANDROID_HOME by default
+# Path to the Android SDK
-# sdk_path = "/path/to/android-sdk"
+sdk_path = "$ANDROID_HOME"
+
+# Path to various versions of the Android NDK
+# Most users will have the latest at $ANDROID_NDK, which is used by default
+# If a version is missing or assigned to None, it is assumed not installed
+ndk_paths = {
+    'r9b': None,
+    'r10d': "$ANDROID_NDK"
+}
 
-# Override the path to the Android NDK, $ANDROID_NDK by default
-# ndk_path = "/path/to/android-ndk"
 # Build tools version to be used
-# build_tools = "21.1.2"
+build_tools = "21.1.2"
 
-# Command for running Ant
+# Command or path to binary for running Ant
-# ant = "/path/to/ant"
 ant = "ant"
 
-# Command for running maven 3
+# Command or path to binary for running maven 3
-# mvn3 = "/path/to/mvn"
 mvn3 = "mvn"
 
-# Command for running Gradle
+# Command or path to binary for running Gradle
-# gradle = "/path/to/gradle"
 gradle = "gradle"
 
 # Set the maximum age (in days) of an index that a client should accept from

fdroidserver/build.py

@@ -452,13 +452,22 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
     """Do a build locally."""
 
     if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
-        if not config['ndk_path']:
+        if not thisbuild['ndk_path']:
-            logging.critical("$ANDROID_NDK is not set!")
+            logging.critical("Android NDK version '%s' could not be found!" % thisbuild['ndk'])
+            logging.critical("Configured versions:")
+            for k, v in config['ndk_paths'].iteritems():
+                if k.endswith("_orig"):
+                    continue
+                logging.critical("  %s: %s" % (k, v))
             sys.exit(3)
-        elif not os.path.isdir(config['sdk_path']):
+        elif not os.path.isdir(thisbuild['ndk_path']):
-            logging.critical("$ANDROID_NDK points to a non-existing directory!")
+            logging.critical("Android NDK '%s' is not a directory!" % thisbuild['ndk_path'])
             sys.exit(3)
 
+    # Set up environment vars that depend on each build
+    for n in ['ANDROID_NDK', 'NDK']:
+        common.env[n] = thisbuild['ndk_path']
+
     # Prepare the source code...
     root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
                                                   build_dir, srclib_dir,
@@ -551,7 +560,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
     # Run a build command if one is required...
     if thisbuild['build']:
         logging.info("Running 'build' commands in %s" % root_dir)
-        cmd = common.replace_config_vars(thisbuild['build'])
+        cmd = common.replace_config_vars(thisbuild['build'], thisbuild)
 
         # Substitute source library paths into commands...
         for name, number, libpath in srclibpaths:
@@ -571,7 +580,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
 
         if jni_components == ['yes']:
             jni_components = ['']
-        cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"]
+        cmd = [os.path.join(thisbuild['ndk_path'], "ndk-build"), "-j1"]
         for d in jni_components:
             if d:
                 logging.info("Building native code in '%s'" % d)
@@ -644,8 +653,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
         modules = bconfig.get('app', 'requirements').split(',')
 
         cmd = 'ANDROIDSDK=' + config['sdk_path']
-        cmd += ' ANDROIDNDK=' + config['ndk_path']
+        cmd += ' ANDROIDNDK=' + thisbuild['ndk_path']
-        cmd += ' ANDROIDNDKVER=r9'
+        cmd += ' ANDROIDNDKVER=' + thisbuild['ndk']
         cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
         cmd += ' VIRTUALENV=virtualenv'
         cmd += ' ./distribute.sh'
@@ -1042,7 +1051,7 @@ def main():
                     # Set up vcs interface and make sure we have the latest code...
                     logging.debug("Getting {0} vcs interface for {1}"
                                   .format(app['Repo Type'], app['Repo']))
-                    vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+                    vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, build)
 
                     first = False
 

fdroidserver/common.py

@@ -41,7 +41,10 @@ env = None
 
 default_config = {
     'sdk_path': "$ANDROID_HOME",
-    'ndk_path': "$ANDROID_NDK",
+    'ndk_paths': {
+        'r9b': None,
+        'r10d': "$ANDROID_NDK"
+    },
     'build_tools': "21.1.2",
     'ant': "ant",
     'mvn3': "mvn",
@@ -82,14 +85,31 @@ def fill_config_defaults(thisconfig):
             thisconfig[k] = v
 
     # Expand paths (~users and $vars)
-    for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
+    def expand_path(path):
+        if path is None:
+            return None
+        orig = path
+        path = os.path.expanduser(path)
+        path = os.path.expandvars(path)
+        if orig == path:
+            return None
+        return path
+
+    for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
         v = thisconfig[k]
-        orig = v
+        exp = expand_path(v)
-        v = os.path.expanduser(v)
+        if exp is not None:
-        v = os.path.expandvars(v)
+            thisconfig[k] = exp
-        if orig != v:
+            thisconfig[k + '_orig'] = v
-            thisconfig[k] = v
+
-            thisconfig[k + '_orig'] = orig
+    for k in ['ndk_paths']:
+        d = thisconfig[k]
+        for k2 in d.copy():
+            v = d[k2]
+            exp = expand_path(v)
+            if exp is not None:
+                thisconfig[k][k2] = exp
+                thisconfig[k][k2 + '_orig'] = v
 
 
 def read_config(opts, config_file='config.py'):
@@ -135,8 +155,6 @@ def read_config(opts, config_file='config.py'):
     env = os.environ
     for n in ['ANDROID_HOME', 'ANDROID_SDK']:
         env[n] = config['sdk_path']
-    for n in ['ANDROID_NDK', 'NDK']:
-        env[n] = config['ndk_path']
 
     for k in ["keystorepass", "keypass"]:
         if k in config:
@@ -165,6 +183,15 @@ def read_config(opts, config_file='config.py'):
     return config
 
 
+def get_ndk_path(version):
+    if version is None:
+        version = 'r10d'  # latest
+    paths = config['ndk_paths']
+    if version not in paths:
+        return None
+    return paths[version]
+
+
 def find_sdk_tools_cmd(cmd):
     '''find a working path to a tool from the Android SDK'''
 
@@ -366,7 +393,7 @@ def getcvname(app):
     return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
 
 
-def getvcs(vcstype, remote, local):
+def getvcs(vcstype, remote, local, build):
     if vcstype == 'git':
         return vcs_git(remote, local)
     if vcstype == 'git-svn':
@@ -378,7 +405,7 @@ def getvcs(vcstype, remote, local):
     if vcstype == 'srclib':
         if local != os.path.join('build', 'srclib', remote):
             raise VCSException("Error: srclib paths are hard-coded!")
-        return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
+        return getsrclib(remote, os.path.join('build', 'srclib'), build, raw=True)
     if vcstype == 'svn':
         raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
     raise VCSException("Invalid vcs type " + vcstype)
@@ -1040,7 +1067,7 @@ class BuildException(FDroidException):
 # Returns the path to it. Normally this is the path to be used when referencing
 # it, which may be a subdirectory of the actual project. If you want the base
 # directory of the project, pass 'basepath=True'.
-def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
+def getsrclib(spec, srclib_dir, build, srclibpaths=[], subdir=None,
               basepath=False, raw=False, prepare=True, preponly=False):
 
     number = None
@@ -1063,7 +1090,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
     sdir = os.path.join(srclib_dir, name)
 
     if not preponly:
-        vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
+        vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir, build)
         vcs.srclib = (name, number, sdir)
         if ref:
             vcs.gotorevision(ref)
@@ -1104,7 +1131,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
     if prepare:
 
         if srclib["Prepare"]:
-            cmd = replace_config_vars(srclib["Prepare"])
+            cmd = replace_config_vars(srclib["Prepare"], build)
 
             p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
             if p.returncode != 0:
@@ -1155,7 +1182,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
 
     # Run an init command if one is required
     if build['init']:
-        cmd = replace_config_vars(build['init'])
+        cmd = replace_config_vars(build['init'], build)
         logging.info("Running 'init' commands in %s" % root_dir)
 
         p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
@@ -1179,7 +1206,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     if build['srclibs']:
         logging.info("Collecting source libraries")
         for lib in build['srclibs']:
-            srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
+            srclibpaths.append(getsrclib(lib, srclib_dir, build, srclibpaths,
                                          preponly=onserver))
 
     for name, number, libpath in srclibpaths:
@@ -1213,10 +1240,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         else:
             props += "sdk.dir=%s\n" % config['sdk_path']
             props += "sdk-location=%s\n" % config['sdk_path']
-        if config['ndk_path']:
+        if build['ndk_path']:
             # Add ndk location
-            props += "ndk.dir=%s\n" % config['ndk_path']
+            props += "ndk.dir=%s\n" % build['ndk_path']
-            props += "ndk-location=%s\n" % config['ndk_path']
+            props += "ndk-location=%s\n" % build['ndk_path']
         # Add java.encoding if necessary
         if build['encoding']:
             props += "java.encoding=%s\n" % build['encoding']
@@ -1336,7 +1363,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     if build['prebuild']:
         logging.info("Running 'prebuild' commands in %s" % root_dir)
 
-        cmd = replace_config_vars(build['prebuild'])
+        cmd = replace_config_vars(build['prebuild'], build)
 
         # Substitute source library paths into prebuild commands
         for name, number, libpath in srclibpaths:
@@ -1764,9 +1791,9 @@ def remove_signing_keys(build_dir):
                     logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
 
 
-def replace_config_vars(cmd):
+def replace_config_vars(cmd, build):
     cmd = cmd.replace('$$SDK$$', config['sdk_path'])
-    cmd = cmd.replace('$$NDK$$', config['ndk_path'])
+    cmd = cmd.replace('$$NDK$$', build['ndk_path'])
     cmd = cmd.replace('$$MVN3$$', config['mvn3'])
     return cmd
 

fdroidserver/init.py

@@ -203,16 +203,8 @@ def main():
     # now that we have a local config.py, read configuration...
     config = common.read_config(options)
 
-    # track down where the Android NDK is
+    # the NDK is optional and there may be multiple versions of it, so it's
-    ndk_path = '/opt/android-ndk'
+    # left for the user to configure
-    if os.path.isdir(config['ndk_path']):
-        ndk_path = config['ndk_path']
-    elif 'ANDROID_NDK' in os.environ.keys():
-        logging.info('using ANDROID_NDK')
-        ndk_path = os.environ['ANDROID_NDK']
-    if os.path.isdir(ndk_path):
-        write_to_config(test_config, 'ndk_path')
-    # the NDK is optional so we don't prompt the user for it if its not found
 
     # find or generate the keystore for the repo signing key. First try the
     # path written in the default config.py.  Then check if the user has
@@ -286,7 +278,7 @@ def main():
     logging.info('  Android SDK:\t\t\t' + config['sdk_path'])
     if aapt:
         logging.info('  Android SDK Build Tools:\t' + os.path.dirname(aapt))
-    logging.info('  Android NDK (optional):\t' + ndk_path)
+    logging.info('  Android NDK r10d (optional):\t$ANDROID_NDK')
     logging.info('  Keystore for signing key:\t' + keystore)
     if repo_keyalias is not None:
         logging.info('  Alias for key in store:\t' + repo_keyalias)

fdroidserver/metadata.py

@@ -25,6 +25,8 @@ import logging
 
 from collections import OrderedDict
 
+import common
+
 srclibs = None
 
 
@@ -100,6 +102,7 @@ flag_defaults = OrderedDict([
     ('scandelete', []),
     ('build', ''),
     ('buildjni', []),
+    ('ndk', 'r9b'),  # defaults to oldest
     ('preassemble', []),
     ('antcommands', None),
     ('novcheck', False),
@@ -560,6 +563,7 @@ def fill_build_defaults(build):
             continue
         build[flag] = value
     build['type'] = get_build_type()
+    build['ndk_path'] = common.get_ndk_path(build['ndk'])
 
 
 # Parse metadata for a single application.

makebuildserver

@@ -102,6 +102,9 @@ cachefiles = [
 
 if config['arch64']:
     cachefiles.extend([
+        ('android-ndk-r10d-linux-x86_64.bin',
+         'https://dl.google.com/android/ndk/android-ndk-r10d-linux-x86_64.bin',
+         '812949f9299afd4b91890863054dc42f6547b6d485211d5f0faca9f286685df6'),
         ('android-ndk-r9b-linux-x86_64.tar.bz2',
          'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
          '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
@@ -110,6 +113,9 @@ if config['arch64']:
          'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1')])
 else:
     cachefiles.extend([
+        ('android-ndk-r10d-linux-x86.bin',
+         'https://dl.google.com/android/ndk/android-ndk-r10d-linux-x86.bin',
+         'c0d07e5ce2fff13b5eb456c10e99527184c9139e798cb7fd1adfadafa65cb696'),
         ('android-ndk-r9b-linux-x86.tar.bz2',
          'https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86.tar.bz2',
          '748104b829dd12afb2fdb3044634963abb24cdb0aad3b26030abe2e9e65bfc81'),