Merge branch 'some-build-fixes' into 'master'

Some build fixes

This includes a couple of fixes discussed with @mvdan including removing the `ANDROID_NDK_HOME` env var from the buildserver and removing the default behavior to force the build-tools version.

See merge request !136

buildserver/provision-android-ndk

@@ -3,26 +3,23 @@
 
 set -e
 
-if [ -z $ANDROID_NDK_HOME ]; then
+NDK_BASE=$1
-    echo "ANDROID_NDK_HOME env var must be set!"
-    exit 1
-fi
 
-test -e $ANDROID_NDK_HOME || mkdir -p $ANDROID_NDK_HOME
+test -e $NDK_BASE || mkdir -p $NDK_BASE
-cd $ANDROID_NDK_HOME
+cd $NDK_BASE
 
-if [ ! -e $ANDROID_NDK_HOME/r9b ]; then
+if [ ! -e $NDK_BASE/r9b ]; then
     if [ `uname -m` == 'x86_64' ] ; then
         SUFFIX='_64'
     else
         SUFFIX=''
     fi
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
+    tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
+    tar xjf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
     mv android-ndk-r9b r9b
 fi
 
-if [ ! -e $ANDROID_NDK_HOME/r10e ]; then
+if [ ! -e $NDK_BASE/r10e ]; then
     if [ `uname -m` == 'x86_64' ] ; then
        SUFFIX='_64'
     else
@@ -32,5 +29,5 @@ if [ ! -e $ANDROID_NDK_HOME/r10e ]; then
     mv android-ndk-r10e r10e
 fi
 
-chmod -R a+rX $ANDROID_NDK_HOME/
+chmod -R a+rX $NDK_BASE/
-find $ANDROID_NDK_HOME/ -type f -executable -print0 | xargs -0 chmod a+x
+find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x

buildserver/provision-android-sdk

@@ -43,7 +43,7 @@ for f in `ls -1 build-tools*.zip`; do
 done
 
 ${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \
-    --filter platform-tools,extra-android-m2repository${cached} <<EOH
+    --filter platform-tools,extra-android-m2repository${cached} > /dev/null <<EOH
 y
 
 EOH

buildserver/setup-env-vars

@@ -10,7 +10,6 @@ bsenv=/etc/profile.d/bsenv.sh
 echo "# generated on "`date` > $bsenv
 
 echo export ANDROID_HOME=$1 >> $bsenv
-echo export ANDROID_NDK_HOME=$2 >> $bsenv
 echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv
 
 chmod 0644 $bsenv

examples/config.py

@@ -25,6 +25,10 @@
 # Build tools version to be used
 # build_tools = "23.0.3"
 
+# Force all build to use the above version of build -tools, good for testing
+# builds without having all of the possible build-tools installed.
+# force_build_tools = True
+
 # Command or path to binary for running Ant
 # ant = "ant"
 
@@ -232,10 +236,10 @@ The repository of older versions of applications from the main demo repository.
 # --server option on dedicated secure build server hosts.
 # build_server_always = True
 
-# By default, fdroid will use YAML and the custom .txt metadata formats. It
+# By default, fdroid will use YAML .yml and the custom .txt metadata formats. It
 # is also possible to have metadata in JSON and XML by adding 'json' and
 # 'xml'.
-# accepted_formats = ['txt', 'yaml']
+# accepted_formats = ['txt', 'yml']
 
 # Limit in number of characters that fields can take up
 # Only the fields listed here are supported, defaults shown

fdroidserver/build.py

@@ -431,8 +431,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
         release_vm()
 
 
-def adapt_gradle(build_dir):
+def force_gradle_build_tools(build_dir, build_tools):
-    filename = 'build.gradle'
     for root, dirs, files in os.walk(build_dir):
         for filename in files:
             if not filename.endswith('.gradle'):
@@ -440,9 +439,9 @@ def adapt_gradle(build_dir):
             path = os.path.join(root, filename)
             if not os.path.isfile(path):
                 continue
-            logging.debug("Adapting %s at %s" % (filename, path))
+            logging.debug("Forcing build-tools %s in %s" % (build_tools, path))
             common.regsub_file(r"""(\s*)buildToolsVersion([\s=]+).*""",
-                               r"""\1buildToolsVersion\2'%s'""" % config['build_tools'],
+                               r"""\1buildToolsVersion\2'%s'""" % build_tools,
                                path)
 
 
@@ -512,9 +511,10 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
 
         gradletasks += ['assemble' + flavours_cmd + 'Release']
 
-        adapt_gradle(build_dir)
+        if config['force_build_tools']:
-        for name, number, libpath in srclibpaths:
+            force_gradle_build_tools(build_dir, config['build_tools'])
-            adapt_gradle(libpath)
+            for name, number, libpath in srclibpaths:
+                force_gradle_build_tools(libpath, config['build_tools'])
 
         cmd = [config['gradle']]
         if build.gradleprops:

fdroidserver/common.py

@@ -59,6 +59,7 @@ default_config = {
         'r10e': "$ANDROID_NDK",
     },
     'build_tools': "23.0.3",
+    'force_build_tools': False,
     'java_paths': None,
     'ant': "ant",
     'mvn3': "mvn",

fdroidserver/metadata.py

@@ -783,9 +783,10 @@ def read_metadata(xref=True):
                                + glob.glob('.fdroid.json')
                                + glob.glob('.fdroid.xml')
                                + glob.glob('.fdroid.yml')):
+        packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+        if packageName in apps:
+            raise MetaDataException("Found multiple metadata files for " + packageName)
         app = parse_metadata(metadatapath)
-        if app.id in apps:
-            raise MetaDataException("Found multiple metadata files for " + app.id)
         check_metadata(app)
         apps[app.id] = app
 

jenkins-build-makebuildserver

@@ -46,7 +46,7 @@ cd $WORKSPACE
 echo "debian_mirror = 'http://ftp.uk.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
 echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
 echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
-./makebuildserver
+./makebuildserver --verbose
 
 # this can be handled in the jenkins job, or here:
 if [ -e fdroiddata ]; then

makebuildserver

@@ -377,10 +377,8 @@ if config['apt_package_cache']:
 
 vagrantfile += """
 
-  config.vm.provision "shell" do |shell|
+  config.vm.provision "shell", path: "setup-env-vars",
-    shell.path = "setup-env-vars"
+    args: ["/home/vagrant/android-sdk"]
-    shell.args = ["/home/vagrant/android-sdk", "/home/vagrant/android-ndk"]
-  end
 
   config.vm.provision :chef_solo do |chef|
     chef.cookbooks_path = "cookbooks"
@@ -397,7 +395,8 @@ vagrantfile += """
   end
 
   config.vm.provision "shell", path: "provision-android-sdk"
-  config.vm.provision "shell", path: "provision-android-ndk"
+  config.vm.provision "shell", path: "provision-android-ndk",
+    args: ["/home/vagrant/android-ndk"]
   config.vm.provision "shell", path: "provision-gradle"
   config.vm.provision "file", source: "gradle",
     destination: "/opt/gradle/bin/gradle"

tests/build.TestCase

@@ -46,7 +46,7 @@ class BuildTest(unittest.TestCase):
                 self.assertTrue(os.path.exists(path))
                 self.assertTrue(os.path.isfile(path))
 
-    def test_adapt_gradle(self):
+    def test_force_gradle_build_tools(self):
         testsbase = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
         if not os.path.exists(testsbase):
             os.makedirs(testsbase)
@@ -54,9 +54,7 @@ class BuildTest(unittest.TestCase):
         shutil.copytree(os.path.join(os.path.dirname(__file__), 'source-files'),
                         os.path.join(testsdir, 'source-files'))
         teststring = 'FAKE_VERSION_FOR_TESTING'
-        fdroidserver.build.config = {}
+        fdroidserver.build.force_gradle_build_tools(testsdir, teststring)
-        fdroidserver.build.config['build_tools'] = teststring
-        fdroidserver.build.adapt_gradle(testsdir)
         pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
         for p in ('source-files/fdroid/fdroidclient/build.gradle',
                   'source-files/Zillode/syncthing-silk/build.gradle',
@@ -67,6 +65,7 @@ class BuildTest(unittest.TestCase):
                 filedata = f.read()
             self.assertIsNotNone(pattern.search(filedata))
 
+
 if __name__ == "__main__":
     parser = optparse.OptionParser()
     parser.add_option("-v", "--verbose", action="store_true", default=False,