From 14b702b23ba27e17827cf9acede353903bb541fc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 13:32:14 +0200 Subject: [PATCH 1/9] githooks: do not run syntax checks on files being deleted --- hooks/pre-commit | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/pre-commit b/hooks/pre-commit index 0f083d00..724a832d 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -23,6 +23,7 @@ else RB_FILES= for f in $files; do + test -e $f || continue case $f in *.py) PY_FILES+=" $f" From 721b3b79a613cd4fd32748cdc04b367ef2eb6e4f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 14:25:22 +0200 Subject: [PATCH 2/9] buildserver: extract NDK rather than executing it Then the NDK archive doesn't need execute permissions. --- buildserver/cookbooks/android-ndk/recipes/default.rb | 6 +++--- .../cookbooks/fdroidbuild-general/recipes/default.rb | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/buildserver/cookbooks/android-ndk/recipes/default.rb b/buildserver/cookbooks/android-ndk/recipes/default.rb index bc704479..506443cb 100644 --- a/buildserver/cookbooks/android-ndk/recipes/default.rb +++ b/buildserver/cookbooks/android-ndk/recipes/default.rb @@ -41,9 +41,9 @@ script "setup-android-ndk-r10e" do else SUFFIX='' fi - chmod u+x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin - /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin x - mv android-ndk-r10e #{ndk_loc}/r10e + cd #{ndk_loc} + 7zr x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin + mv android-ndk-r10e r10e " not_if "test -d #{ndk_loc}/r10e" end diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb index 3d7ca78d..6cde2f92 100644 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -54,6 +54,7 @@ end openjdk-7-jdk openjdk-8-jdk optipng + p7zip pandoc perlmagick pkg-config From e449d2f5839805071c0837dbd6ff401bebbe12c3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 13:09:57 +0200 Subject: [PATCH 3/9] buildserver: setup env vars using standard script bash provides a standard file location for a script to be run when the shell starts: /etc/profile.d/ This converts the scattered bits of code for making ~/.bsenv into a single provisioning script to generate /etc/profile.d/bsenv.sh, which gets automatically executed when bash starts --- .../fdroidbuild-general/recipes/default.rb | 6 ------ buildserver/cookbooks/gradle/recipes/default.rb | 6 ------ buildserver/setup-env-vars | 16 ++++++++++++++++ fdroidserver/build.py | 2 +- makebuildserver | 6 ++++++ 5 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 buildserver/setup-env-vars diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb index 6cde2f92..f9f81fe0 100644 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -97,12 +97,6 @@ easy_install_package "compare-locales" do action :install end -execute "add-bsenv" do - user user - command "echo \". ./.bsenv \" >> /home/#{user}/.bashrc" - not_if "grep bsenv /home/#{user}/.bashrc" -end - execute "set-default-java" do command "update-java-alternatives --set java-1.7.0-openjdk-i386" end diff --git a/buildserver/cookbooks/gradle/recipes/default.rb b/buildserver/cookbooks/gradle/recipes/default.rb index 74dba2c4..2c2a4956 100644 --- a/buildserver/cookbooks/gradle/recipes/default.rb +++ b/buildserver/cookbooks/gradle/recipes/default.rb @@ -40,9 +40,3 @@ EOF chmod a+x /opt/gradle/bin/gradle " end - -execute "add-android-ndk-path" do - user user - command "echo \"export PATH=\\$PATH:/opt/gradle/bin #PATH-GRADLE\" >> /home/#{user}/.bsenv" - not_if "grep PATH-GRADLE /home/#{user}/.bsenv" -end diff --git a/buildserver/setup-env-vars b/buildserver/setup-env-vars new file mode 100644 index 00000000..83d433eb --- /dev/null +++ b/buildserver/setup-env-vars @@ -0,0 +1,16 @@ +#!/bin/sh +# +# sets up the environment vars needed by the build process + +set -e +set -x + +bsenv=/etc/profile.d/bsenv.sh + +echo "# generated on "`date` > $bsenv + +echo export ANDROID_HOME=$1 >> $bsenv +echo export ANDROID_NDK_HOME=$2 >> $bsenv +echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv + +chmod 0644 $bsenv diff --git a/fdroidserver/build.py b/fdroidserver/build.py index a387d13b..8376cadf 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -388,7 +388,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force): if options.verbose: cmdline += ' --verbose' cmdline += " %s:%s" % (app.id, build.vercode) - chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"') + chan.exec_command('bash -c "' + cmdline + '"') output = bytes() while not chan.exit_status_ready(): while chan.recv_ready(): diff --git a/makebuildserver b/makebuildserver index cff8b64e..dbb51005 100755 --- a/makebuildserver +++ b/makebuildserver @@ -373,6 +373,12 @@ if config['apt_package_cache']: """.format(aptcachedir) vagrantfile += """ + + config.vm.provision "shell" do |shell| + shell.path = "setup-env-vars" + shell.args = ["/home/vagrant/android-sdk", "/home/vagrant/android-ndk"] + end + config.vm.provision :chef_solo do |chef| chef.cookbooks_path = "cookbooks" chef.log_level = :debug From e47396b4035fe0f82d85bd0ec38ef38169ccf6fb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 14 Jun 2016 16:34:13 +0200 Subject: [PATCH 4/9] buildserver: use `android update sdk` to install Android SDK `android update sdk --no-ui` is the standard command line tool for installing the Android SDK. By symlinking into the $ANDROID_HOME/temp dir, the cached files can still be used. This converts the chef recipe to a vagrant shell provisioning script since it was all bash anyway. Some file names no longer officially have a -linux in them, so those were changed to keep the cache working with the default filename. --- .../cookbooks/android-sdk/recipes/default.rb | 79 ------------------- buildserver/provision-android-sdk | 53 +++++++++++++ makebuildserver | 21 ++--- 3 files changed, 65 insertions(+), 88 deletions(-) delete mode 100644 buildserver/cookbooks/android-sdk/recipes/default.rb create mode 100644 buildserver/provision-android-sdk diff --git a/buildserver/cookbooks/android-sdk/recipes/default.rb b/buildserver/cookbooks/android-sdk/recipes/default.rb deleted file mode 100644 index d5a87170..00000000 --- a/buildserver/cookbooks/android-sdk/recipes/default.rb +++ /dev/null @@ -1,79 +0,0 @@ - -sdk_loc = node[:settings][:sdk_loc] -user = node[:settings][:user] - -script "setup-android-sdk" do - timeout 14400 - interpreter "bash" - user user - cwd "/tmp" - code " - tools=`ls -1 /vagrant/cache/tools_*.zip | sort -n | tail -1` - unzip $tools - mkdir #{sdk_loc} - mkdir #{sdk_loc}/platforms - mkdir #{sdk_loc}/build-tools - mv tools #{sdk_loc}/ - " - not_if "test -d #{sdk_loc}" -end - -execute "add-android-sdk-path" do - user user - path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools" - command "echo \"export PATH=\\$PATH:#{path} #PATH-SDK\" >> /home/#{user}/.bsenv" - not_if "grep PATH-SDK /home/#{user}/.bsenv" -end - -script "add_android_packages" do - interpreter "bash" - user user - code " - #{sdk_loc}/tools/android update sdk --no-ui --all --filter platform-tools,extra-android-m2repository < { - :sdk_loc => "/home/vagrant/android-sdk", :ndk_loc => "/home/vagrant/android-ndk", :debian_mirror => "%s", :ubuntu_trusty => "%s", @@ -392,11 +393,13 @@ vagrantfile += """ } } chef.add_recipe "fdroidbuild-general" - chef.add_recipe "android-sdk" chef.add_recipe "android-ndk" chef.add_recipe "gradle" chef.add_recipe "kivy" end + + config.vm.provision "shell", path: "provision-android-sdk" + end """ % (config['debian_mirror'], str('14.04' in os.uname()[3]).lower()) From 02a835ff95e0ce6ac167205f50c6439159afd4b6 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 15:02:37 +0200 Subject: [PATCH 5/9] buildserver: unpack NDK with a provisioning shell script This makes the process closer to ./jenkins-build scripts and .gitlab-ci.yml files. Hopefully it uses less RAM than chef too. --- .../cookbooks/android-ndk/recipes/default.rb | 50 ------------------- buildserver/provision-android-ndk | 36 +++++++++++++ makebuildserver | 3 +- 3 files changed, 37 insertions(+), 52 deletions(-) delete mode 100644 buildserver/cookbooks/android-ndk/recipes/default.rb create mode 100644 buildserver/provision-android-ndk diff --git a/buildserver/cookbooks/android-ndk/recipes/default.rb b/buildserver/cookbooks/android-ndk/recipes/default.rb deleted file mode 100644 index 506443cb..00000000 --- a/buildserver/cookbooks/android-ndk/recipes/default.rb +++ /dev/null @@ -1,50 +0,0 @@ - -ndk_loc = node[:settings][:ndk_loc] -user = node[:settings][:user] - -script "setup-android-ndk" do - timeout 14400 - interpreter "bash" - user node[:settings][:user] - cwd "/tmp" - code " - mkdir -p #{ndk_loc} - " -end - -script "setup-android-ndk-r9b" do - timeout 14400 - interpreter "bash" - user node[:settings][:user] - cwd "/tmp" - code " - if [ `uname -m` == 'x86_64' ] ; then - SUFFIX='_64' - else - SUFFIX='' - fi - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 - tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 - mv android-ndk-r9b #{ndk_loc}/r9b - " - not_if "test -d #{ndk_loc}/r9b" -end - -script "setup-android-ndk-r10e" do - timeout 14400 - interpreter "bash" - user node[:settings][:user] - cwd "/tmp" - code " - if [ `uname -m` == 'x86_64' ] ; then - SUFFIX='_64' - else - SUFFIX='' - fi - cd #{ndk_loc} - 7zr x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin - mv android-ndk-r10e r10e - " - not_if "test -d #{ndk_loc}/r10e" -end - diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk new file mode 100644 index 00000000..3c9b0a6a --- /dev/null +++ b/buildserver/provision-android-ndk @@ -0,0 +1,36 @@ +#!/bin/bash +# + +set -e + +if [ -z $ANDROID_NDK_HOME ]; then + echo "ANDROID_NDK_HOME env var must be set!" + exit 1 +fi + +test -e $ANDROID_NDK_HOME || mkdir -p $ANDROID_NDK_HOME +cd $ANDROID_NDK_HOME + +if [ ! -e $ANDROID_NDK_HOME/r9b ]; then + if [ `uname -m` == 'x86_64' ] ; then + SUFFIX='_64' + else + SUFFIX='' + fi + tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX.tar.bz2 + tar jxvf /vagrant/cache/android-ndk-r9b-linux-x86$SUFFIX-legacy-toolchains.tar.bz2 + mv android-ndk-r9b r9b +fi + +if [ ! -e $ANDROID_NDK_HOME/r10e ]; then + if [ `uname -m` == 'x86_64' ] ; then + SUFFIX='_64' + else + SUFFIX='' + fi + 7zr x /vagrant/cache/android-ndk-r10e-linux-x86$SUFFIX.bin > /dev/null + mv android-ndk-r10e r10e +fi + +chmod -R a+rX $ANDROID_NDK_HOME/ +find $ANDROID_NDK_HOME/ -type f -executable -print0 | xargs -0 chmod a+x diff --git a/makebuildserver b/makebuildserver index 2f9a8ad3..bfe7f149 100755 --- a/makebuildserver +++ b/makebuildserver @@ -386,19 +386,18 @@ vagrantfile += """ chef.log_level = :debug chef.json = { :settings => { - :ndk_loc => "/home/vagrant/android-ndk", :debian_mirror => "%s", :ubuntu_trusty => "%s", :user => "vagrant" } } chef.add_recipe "fdroidbuild-general" - chef.add_recipe "android-ndk" chef.add_recipe "gradle" chef.add_recipe "kivy" end config.vm.provision "shell", path: "provision-android-sdk" + config.vm.provision "shell", path: "provision-android-ndk" end """ % (config['debian_mirror'], From 6ea2508127519e33a44fc32500d0dee5aaf96a94 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 15:25:57 +0200 Subject: [PATCH 6/9] buildserver: unpack gradle versions with provisioning shell script This was not using anything special from chef, so do it in a shell script instead. This makes the script easier for the python/shell people, and probably uses less memory, since chef is a memory hog. This might even make the provision go faster since it uploads the whole script as a file to the VM, then runs it there. I think chef sends each command via SSH. --- .../cookbooks/gradle/recipes/default.rb | 42 ------------------- .../{cookbooks/gradle/recipes => }/gradle | 0 buildserver/provision-gradle | 20 +++++++++ makebuildserver | 4 +- 4 files changed, 23 insertions(+), 43 deletions(-) delete mode 100644 buildserver/cookbooks/gradle/recipes/default.rb rename buildserver/{cookbooks/gradle/recipes => }/gradle (100%) create mode 100644 buildserver/provision-gradle diff --git a/buildserver/cookbooks/gradle/recipes/default.rb b/buildserver/cookbooks/gradle/recipes/default.rb deleted file mode 100644 index 2c2a4956..00000000 --- a/buildserver/cookbooks/gradle/recipes/default.rb +++ /dev/null @@ -1,42 +0,0 @@ - -user = node[:settings][:user] - -gradle_script = IO.read(File.join( - File.expand_path(File.dirname(__FILE__)), "gradle")) - -script "add-gradle-bindir" do - cwd "/tmp" - interpreter "bash" - code "mkdir -p /opt/gradle/bin" - not_if "test -d /opt/gradle/bin" -end - -script "add-gradle-verdir" do - cwd "/tmp" - interpreter "bash" - code "mkdir -p /opt/gradle/versions" - not_if "test -d /opt/gradle/versions" -end - -%w{1.4 1.6 1.7 1.8 1.9 1.10 1.11 1.12 2.1 2.2.1 2.3 2.4 2.5 2.6 2.7 2.8 2.9 2.10 2.11 2.12 2.13}.each do |ver| - script "install-gradle-#{ver}" do - cwd "/tmp" - interpreter "bash" - code " - unzip /vagrant/cache/gradle-#{ver}-bin.zip - mv gradle-#{ver} /opt/gradle/versions/#{ver} - " - not_if "test -d /opt/gradle/versions/#{ver}" - end -end - -script "add-gradle-wrapper" do - cwd "/tmp" - interpreter "bash" - code " - cat << \"EOF\" > /opt/gradle/bin/gradle -#{gradle_script} -EOF - chmod a+x /opt/gradle/bin/gradle - " -end diff --git a/buildserver/cookbooks/gradle/recipes/gradle b/buildserver/gradle similarity index 100% rename from buildserver/cookbooks/gradle/recipes/gradle rename to buildserver/gradle diff --git a/buildserver/provision-gradle b/buildserver/provision-gradle new file mode 100644 index 00000000..124b99aa --- /dev/null +++ b/buildserver/provision-gradle @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions +cd /opt/gradle/versions +for f in /vagrant/cache/gradle-*.zip; do + ver=`echo $f | sed 's,.*gradle-\([0-9][0-9.]*\).*\.zip,\1,'` + if [ ! -d /opt/gradle/versions/${ver} ]; then + unzip -qq $f + mv gradle-${ver} /opt/gradle/versions/${ver} + fi +done + +chmod -R a+rX /opt/gradle + +test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin +touch /opt/gradle/bin/gradle +chown vagrant.vagrant /opt/gradle/bin/gradle +chmod 0755 /opt/gradle/bin/gradle diff --git a/makebuildserver b/makebuildserver index bfe7f149..d44e559d 100755 --- a/makebuildserver +++ b/makebuildserver @@ -392,12 +392,14 @@ vagrantfile += """ } } chef.add_recipe "fdroidbuild-general" - chef.add_recipe "gradle" chef.add_recipe "kivy" end config.vm.provision "shell", path: "provision-android-sdk" config.vm.provision "shell", path: "provision-android-ndk" + config.vm.provision "shell", path: "provision-gradle" + config.vm.provision "file", source: "gradle", + destination: "/opt/gradle/bin/gradle" end """ % (config['debian_mirror'], From d0bb6f73bfeaf5b74b4146a86e1470976b9a4d80 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 12:47:16 +0200 Subject: [PATCH 7/9] buildserver: /vagrant/cache writeable only by root Prevent build processes from modifying the cache, it is only needed during provisioning anyway. A malicious build could still use sudo to change the cache, but this is more to prevent mistaken modifications. --- makebuildserver | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makebuildserver b/makebuildserver index d44e559d..0f5cb86e 100755 --- a/makebuildserver +++ b/makebuildserver @@ -363,7 +363,8 @@ if 'aptproxy' in config and config['aptproxy']: # does not need a custom mount if cachedir != 'buildserver/cache': vagrantfile += """ - config.vm.synced_folder '{0}', '/vagrant/cache' + config.vm.synced_folder '{0}', '/vagrant/cache', + owner: 'root', group: 'root', create: true """.format(cachedir) # cache .deb packages on the host via a mount trick From c11edc302207b717c53a604c71fc32166cb6e40e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 15:39:49 +0200 Subject: [PATCH 8/9] include new provisioning shell scripts in syntax checks --- hooks/pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/pre-commit b/hooks/pre-commit index 724a832d..9f0dcb77 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -11,7 +11,7 @@ if [ -z "$files" ]; then PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" - BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" + BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*" RB_FILES="buildserver/cookbooks/*/recipes/*.rb" else # if actually committing right now, then only run on the files From 8e57f2737d97b9ce9bf23e118162b6e481617c51 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 15 Jun 2016 16:34:10 +0200 Subject: [PATCH 9/9] invalidate apkcache if the metadata version is different When the metadata changes, different things will be stored about each APK. So invalidate the cached info parsed from APKs if the cache's metadata version does not match the metadata version of the currently running tools. --- fdroidserver/update.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index cd75449d..7cd18f41 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -44,6 +44,8 @@ from . import metadata from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen from .metadata import MetaDataException +METADATA_VERSION = 16 + screen_densities = ['640', '480', '320', '240', '160', '120'] all_screen_densities = ['0'] + screen_densities @@ -811,7 +813,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): for mirror in config.get('mirrors', []): addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel) - repoel.setAttribute("version", "16") + repoel.setAttribute("version", str(METADATA_VERSION)) repoel.setAttribute("timestamp", str(int(time.time()))) nosigningkey = False @@ -1224,6 +1226,8 @@ def main(): if not options.clean and os.path.exists(apkcachefile): with open(apkcachefile, 'rb') as cf: apkcache = pickle.load(cf, encoding='utf-8') + if apkcache.get("METADATA_VERSION") != METADATA_VERSION: + apkcache = {} else: apkcache = {} @@ -1364,6 +1368,7 @@ def main(): f.write(data) if cachechanged: + apkcache["METADATA_VERSION"] = METADATA_VERSION with open(apkcachefile, 'wb') as cf: pickle.dump(apkcache, cf)