Merge branch 'use-android-update-sdk' into 'master'

switch SDK/NDK/gradle buildserver provisioning to shell scripts

This converts the SDK, NDK, and gradle chef recipes to vagrant provisioning shell scripts.   Those recipes were just shell scripts anyway, forced into that ugly ruby/chef syntax.  Hopefully this will make things easier to maintain since simple bash is easier for most devs than ruby/chef. Also, it might speed up the provisioning a little bit since the whole script is sent to the VM then executed, rather than sent line-by-line.  Additionally, the SDK components are now installed using `android update sdk` so we do not need to duplicate Google's crazy kludges with that stuff.

See merge request !132

buildserver/cookbooks/android-ndk/recipes/default.rb

@@ -1,50 +0,0 @@
-
-ndk_loc = node[:settings][:ndk_loc]
-user = node[:settings][:user]
-
-script "setup-android-ndk" do
-  timeout 14400
-  interpreter "bash"
-  user node[:settings][:user]
-  cwd "/tmp"
-  code "
-    mkdir -p #{ndk_loc}
-  "
-end
-
-script "setup-android-ndk-r9b" do
-  timeout 14400
-  interpreter "bash"
-  user node[:settings][:user]
-  cwd "/tmp"
-  code "
-    if [ `uname -m` == 'x86_64' ] ; then
-       SUFFIX='_64'
-    else
-       SUFFIX=''
-    fi
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
-    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
-    mv android-ndk-r9b #{ndk_loc}/r9b
-  "
-  not_if "test -d #{ndk_loc}/r9b"
-end
-
-script "setup-android-ndk-r10e" do
-  timeout 14400
-  interpreter "bash"
-  user node[:settings][:user]
-  cwd "/tmp"
-  code "
-    if [ `uname -m` == 'x86_64' ] ; then
-       SUFFIX='_64'
-    else
-       SUFFIX=''
-    fi
-    chmod u+x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin
-    /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin x
-    mv android-ndk-r10e #{ndk_loc}/r10e
-  "
-  not_if "test -d #{ndk_loc}/r10e"
-end
-

buildserver/cookbooks/android-sdk/recipes/default.rb

@@ -1,79 +0,0 @@
-
-sdk_loc = node[:settings][:sdk_loc]
-user = node[:settings][:user]
-
-script "setup-android-sdk" do
-  timeout 14400
-  interpreter "bash"
-  user user
-  cwd "/tmp"
-  code "
-    tools=`ls -1 /vagrant/cache/tools_*.zip | sort -n | tail -1`
-    unzip $tools
-    mkdir #{sdk_loc}
-    mkdir #{sdk_loc}/platforms
-    mkdir #{sdk_loc}/build-tools
-    mv tools #{sdk_loc}/
-  "
-  not_if "test -d #{sdk_loc}"
-end
-
-execute "add-android-sdk-path" do
-  user user
-  path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools"
-  command "echo \"export PATH=\\$PATH:#{path} #PATH-SDK\" >> /home/#{user}/.bsenv"
-  not_if "grep PATH-SDK /home/#{user}/.bsenv"
-end
-
-script "add_android_packages" do
-  interpreter "bash"
-  user user
-  code "
-    #{sdk_loc}/tools/android update sdk --no-ui --all --filter platform-tools,extra-android-m2repository <<X
-y
-
-X
-    "
-end
-
-script "add-platforms" do
-  interpreter "bash"
-  user user
-  code "
-    rm -rf current-platform
-    mkdir current-platform
-    cd current-platform
-    for f in `ls -1 /vagrant/cache/android-[0-9]*.zip /vagrant/cache/platform-[0-9]*.zip`; do
-      unzip $f
-      sdk=`sed -n 's,^ro.build.version.sdk=,,p' */build.prop`
-      rm -rf #{sdk_loc}/platforms/android-$sdk
-      mv * #{sdk_loc}/platforms/android-$sdk
-    done
-  "
-end
-
-script "add_build_tools" do
-  interpreter "bash"
-  user user
-  code "
-    rm -rf current-build-tools
-    mkdir current-build-tools
-    cd current-build-tools
-    for ver in 17 18.0.1 18.1 18.1.1 19 19.0.1 19.0.2 19.0.3 19.1 20 21 21.0.1 21.0.2 21.1 21.1.1 21.1.2 22 22.0.1 23 23.0.1 23.0.2 23.0.3; do
-        unzip /vagrant/cache/build-tools_r${ver}-linux.zip
-        case `echo ${ver} | wc -c` in
-            3)
-                dirver=${ver}.0.0
-                ;;
-            5)
-                dirver=${ver}.0
-                ;;
-            7)
-                dirver=${ver}
-                ;;
-        esac
-        rm -rf #{sdk_loc}/build-tools/${dirver}
-        mv android-*/ #{sdk_loc}/build-tools/${dirver}
-    done
-  "
-end

buildserver/cookbooks/fdroidbuild-general/recipes/default.rb

@@ -54,6 +54,7 @@ end
     openjdk-7-jdk
     openjdk-8-jdk
     optipng
+    p7zip
     pandoc
     perlmagick
     pkg-config
@@ -96,12 +97,6 @@ easy_install_package "compare-locales" do
   action :install
 end
 
-execute "add-bsenv" do
-  user user
-  command "echo \". ./.bsenv \" >> /home/#{user}/.bashrc"
-  not_if "grep bsenv /home/#{user}/.bashrc"
-end
-
 execute "set-default-java" do
   command "update-java-alternatives --set java-1.7.0-openjdk-i386"
 end

buildserver/cookbooks/gradle/recipes/default.rb

@@ -1,48 +0,0 @@
-
-user = node[:settings][:user]
-
-gradle_script = IO.read(File.join(
-	File.expand_path(File.dirname(__FILE__)), "gradle"))
-
-script "add-gradle-bindir" do
-  cwd "/tmp"
-  interpreter "bash"
-  code "mkdir -p /opt/gradle/bin"
-  not_if "test -d /opt/gradle/bin"
-end
-
-script "add-gradle-verdir" do
-  cwd "/tmp"
-  interpreter "bash"
-  code "mkdir -p /opt/gradle/versions"
-  not_if "test -d /opt/gradle/versions"
-end
-
-%w{1.4 1.6 1.7 1.8 1.9 1.10 1.11 1.12 2.1 2.2.1 2.3 2.4 2.5 2.6 2.7 2.8 2.9 2.10 2.11 2.12 2.13}.each do |ver|
-  script "install-gradle-#{ver}" do
-    cwd "/tmp"
-    interpreter "bash"
-    code "
-      unzip /vagrant/cache/gradle-#{ver}-bin.zip
-      mv gradle-#{ver} /opt/gradle/versions/#{ver}
-    "
-    not_if "test -d /opt/gradle/versions/#{ver}"
-  end
-end
-
-script "add-gradle-wrapper" do
-  cwd "/tmp"
-  interpreter "bash"
-  code "
-    cat << \"EOF\" > /opt/gradle/bin/gradle
-#{gradle_script}
-EOF
-    chmod a+x /opt/gradle/bin/gradle
-  "
-end
-
-execute "add-android-ndk-path" do
-  user user
-  command "echo \"export PATH=\\$PATH:/opt/gradle/bin #PATH-GRADLE\" >> /home/#{user}/.bsenv"
-  not_if "grep PATH-GRADLE /home/#{user}/.bsenv"
-end

buildserver/provision-android-ndk

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+
+set -e
+
+if [ -z $ANDROID_NDK_HOME ]; then
+    echo "ANDROID_NDK_HOME env var must be set!"
+    exit 1
+fi
+
+test -e $ANDROID_NDK_HOME || mkdir -p $ANDROID_NDK_HOME
+cd $ANDROID_NDK_HOME
+
+if [ ! -e $ANDROID_NDK_HOME/r9b ]; then
+    if [ `uname -m` == 'x86_64' ] ; then
+        SUFFIX='_64'
+    else
+        SUFFIX=''
+    fi
+    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2
+    tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2
+    mv android-ndk-r9b r9b
+fi
+
+if [ ! -e $ANDROID_NDK_HOME/r10e ]; then
+    if [ `uname -m` == 'x86_64' ] ; then
+       SUFFIX='_64'
+    else
+       SUFFIX=''
+    fi
+    7zr x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin > /dev/null
+    mv android-ndk-r10e r10e
+fi
+
+chmod -R a+rX $ANDROID_NDK_HOME/
+find $ANDROID_NDK_HOME/ -type f -executable -print0 | xargs -0 chmod a+x

buildserver/provision-android-sdk

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+
+set -e
+
+if [ -z $ANDROID_HOME ]; then
+    echo "ANDROID_HOME env var must be set!"
+    exit 1
+fi
+
+# TODO remove the rm, this should work with an existing ANDROID_HOME
+if [ ! -x $ANDROID_HOME/tools/android ]; then
+    rm -rf $ANDROID_HOME
+    mkdir ${ANDROID_HOME}
+    mkdir ${ANDROID_HOME}/temp
+    mkdir ${ANDROID_HOME}/platforms
+    mkdir ${ANDROID_HOME}/build-tools
+    cd $ANDROID_HOME
+
+    tools=`ls -1 /vagrant/cache/tools_*.zip | sort -n | tail -1`
+    unzip -qq $tools
+fi
+
+cd /vagrant/cache
+
+# make hard links for `android update sdk` to use and delete
+for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do
+    rm -f ${ANDROID_HOME}/temp/$f
+    ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/
+done
+
+# install all cached platforms
+cached=""
+for f in `ls -1 android-[0-9]*.zip platform-[0-9]*.zip`; do
+    sdk=`unzip -c $f "*/build.prop" | sed -n 's,^ro.build.version.sdk=,,p'`
+    cached=,android-${sdk}${cached}
+done
+
+# install all cached build-tools
+for f in `ls -1 build-tools*.zip`; do
+    ver=`unzip -c $f "*/source.properties" | sed -n 's,^Pkg.Revision=,,p'`
+    cached=,build-tools-${ver}${cached}
+done
+
+${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \
+    --filter platform-tools,extra-android-m2repository${cached} <<EOH
+y
+
+EOH
+
+
+chmod -R a+rX $ANDROID_HOME/
+find $ANDROID_HOME/ -type f -executable -print0 | xargs -0 chmod a+x

buildserver/provision-gradle

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions
+cd /opt/gradle/versions
+for f in /vagrant/cache/gradle-*.zip; do
+    ver=`echo $f | sed 's,.*gradle-\([0-9][0-9.]*\).*\.zip,\1,'`
+    if [ ! -d /opt/gradle/versions/${ver} ]; then
+        unzip -qq $f
+        mv gradle-${ver} /opt/gradle/versions/${ver}
+    fi
+done
+
+chmod -R a+rX /opt/gradle
+
+test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin
+touch /opt/gradle/bin/gradle
+chown vagrant.vagrant /opt/gradle/bin/gradle
+chmod 0755 /opt/gradle/bin/gradle

buildserver/setup-env-vars

@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# sets up the environment vars needed by the build process
+
+set -e
+set -x
+
+bsenv=/etc/profile.d/bsenv.sh
+
+echo "# generated on "`date` > $bsenv
+
+echo export ANDROID_HOME=$1 >> $bsenv
+echo export ANDROID_NDK_HOME=$2 >> $bsenv
+echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv
+
+chmod 0644 $bsenv

fdroidserver/build.py

@@ -388,7 +388,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
         if options.verbose:
             cmdline += ' --verbose'
         cmdline += " %s:%s" % (app.id, build.vercode)
-        chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
+        chan.exec_command('bash -c "' + cmdline + '"')
         output = bytes()
         while not chan.exit_status_ready():
             while chan.recv_ready():

fdroidserver/update.py

@@ -44,6 +44,8 @@ from . import metadata
 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
 from .metadata import MetaDataException
 
+METADATA_VERSION = 16
+
 screen_densities = ['640', '480', '320', '240', '160', '120']
 
 all_screen_densities = ['0'] + screen_densities
@@ -811,7 +813,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
         for mirror in config.get('mirrors', []):
             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 
-    repoel.setAttribute("version", "16")
+    repoel.setAttribute("version", str(METADATA_VERSION))
     repoel.setAttribute("timestamp", str(int(time.time())))
 
     nosigningkey = False
@@ -1224,6 +1226,8 @@ def main():
     if not options.clean and os.path.exists(apkcachefile):
         with open(apkcachefile, 'rb') as cf:
             apkcache = pickle.load(cf, encoding='utf-8')
+        if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
+            apkcache = {}
     else:
         apkcache = {}
 
@@ -1364,6 +1368,7 @@ def main():
                 f.write(data)
 
     if cachechanged:
+        apkcache["METADATA_VERSION"] = METADATA_VERSION
         with open(apkcachefile, 'wb') as cf:
             pickle.dump(apkcache, cf)
 

hooks/pre-commit

@@ -11,7 +11,7 @@ if [ -z "$files" ]; then
     PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py"
     PY_TEST_FILES="tests/*.TestCase"
     SH_FILES="hooks/pre-commit"
-    BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion"
+    BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*"
     RB_FILES="buildserver/cookbooks/*/recipes/*.rb"
 else
     # if actually committing right now, then only run on the files
@@ -23,6 +23,7 @@ else
     RB_FILES=
 
     for f in $files; do
+        test -e $f || continue
         case $f in
             *.py)
                 PY_FILES+=" $f"

makebuildserver

@@ -89,6 +89,8 @@ if not os.path.exists(cachedir):
 cachefiles = [
     ('https://dl.google.com/android/repository/tools_r25.1.7-linux.zip',
      '3ca053600a86a5a64d5571edfbb1dad27f2bda3bfd2d38e2fe54322610b1ef0b'),
+    ('https://dl.google.com/android/repository/android_m2repository_r32.zip',
+     'a6a8d7ffb153161f26d5fdebfa9aa1c9c84b29c62851fffff2cdfad9e094b13b'),
     ('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
      '85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
     ('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
@@ -97,19 +99,19 @@ cachefiles = [
      'e70e2151b49613f23f40828c771ab85e241eed361cab037c6312df77f2612f0a'),
     ('https://dl.google.com/android/repository/android-2.0.1_r01-linux.zip',
      'f47b46177b17f6368461f85bc2a27d0d2c437929f588ea27105712bc3185f664'),
-    ('https://dl.google.com/android/repository/android-2.1_r03-linux.zip',
+    ('https://dl.google.com/android/repository/android-2.1_r03.zip',
      'b9cc140a9b879586181b22cfc7d4aa18b979251e16e9b17771c5d0acb71ba940'),
-    ('https://dl.google.com/android/repository/android-2.2_r03-linux.zip',
+    ('https://dl.google.com/android/repository/android-2.2_r03.zip',
      '7c9ea1bd7cb225504bd085d7c93ae27d52bd88d29b621d28108f82fef68177c0'),
-    ('https://dl.google.com/android/repository/android-2.3.1_r02-linux.zip',
+    ('https://dl.google.com/android/repository/android-2.3.1_r02.zip',
      'b2ab4896d0a4857e4f688f69eb08b0e1a8074709d4445a92a83ece7ec7cd198c'),
-    ('https://dl.google.com/android/repository/android-2.3.3_r02-linux.zip',
+    ('https://dl.google.com/android/repository/android-2.3.3_r02.zip',
      '54bdb0f1ca06ba5747061ddeea20f431af72c448334fd4d3d7f84ea2ccd29fea'),
-    ('https://dl.google.com/android/repository/android-3.0_r02-linux.zip',
+    ('https://dl.google.com/android/repository/android-3.0_r02.zip',
      '1cacae7b6e1b5a5d73c06f5d29d2ea92d16674df8fd5507681290e77d1647a1c'),
-    ('https://dl.google.com/android/repository/android-3.1_r03-linux.zip',
+    ('https://dl.google.com/android/repository/android-3.1_r03.zip',
      '7570c86a86488a146aa2141a65a24d81800959c1907ff4f1d2c13bbafab230c5'),
-    ('https://dl.google.com/android/repository/android-3.2_r01-linux.zip',
+    ('https://dl.google.com/android/repository/android-3.2_r01.zip',
      'ff6b26ad34d7060a72ba504b0314cef8ba3138005561705adec5ad470a073d9b'),
     ('https://dl.google.com/android/repository/android-14_r04.zip',
      'da1af15c77ba41d062eb6d0ef5921cc424ab6167587033b830609d65f04802b6'),
@@ -361,7 +363,8 @@ if 'aptproxy' in config and config['aptproxy']:
 # does not need a custom mount
 if cachedir != 'buildserver/cache':
     vagrantfile += """
-  config.vm.synced_folder '{0}', '/vagrant/cache'
+  config.vm.synced_folder '{0}', '/vagrant/cache',
+    owner: 'root', group: 'root', create: true
 """.format(cachedir)
 
 # cache .deb packages on the host via a mount trick
@@ -373,24 +376,32 @@ if config['apt_package_cache']:
 """.format(aptcachedir)
 
 vagrantfile += """
+
+  config.vm.provision "shell" do |shell|
+    shell.path = "setup-env-vars"
+    shell.args = ["/home/vagrant/android-sdk", "/home/vagrant/android-ndk"]
+  end
+
   config.vm.provision :chef_solo do |chef|
     chef.cookbooks_path = "cookbooks"
     chef.log_level = :debug
     chef.json = {
       :settings => {
-        :sdk_loc => "/home/vagrant/android-sdk",
-        :ndk_loc => "/home/vagrant/android-ndk",
         :debian_mirror => "%s",
         :ubuntu_trusty => "%s",
         :user => "vagrant"
       }
     }
     chef.add_recipe "fdroidbuild-general"
-    chef.add_recipe "android-sdk"
-    chef.add_recipe "android-ndk"
-    chef.add_recipe "gradle"
     chef.add_recipe "kivy"
   end
+
+  config.vm.provision "shell", path: "provision-android-sdk"
+  config.vm.provision "shell", path: "provision-android-ndk"
+  config.vm.provision "shell", path: "provision-gradle"
+  config.vm.provision "file", source: "gradle",
+    destination: "/opt/gradle/bin/gradle"
+
 end
 """ % (config['debian_mirror'],
        str('14.04' in os.uname()[3]).lower())