From 3fc2a99d7110c4263e2354ebcbf0aeb4637a79fc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 1 Sep 2015 11:23:23 +0200 Subject: [PATCH 1/4] `fdroid --version` for installed releases and running from git This will report the version embedded in the module if it is installed, and will report `git describe` if being run from git. If someone installs from git using pip, this will probably report the version in setup.py, which will be wrong. But that is not a documented install method, and I haven't heard of anyone using it. The recommended way is to run straight from git. --- completion/bash-completion | 4 ++-- fdroid | 32 +++++++++++++++++++++++++++++--- tests/run-tests | 6 ++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index f4dc01d8..0fd88d3b 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -84,10 +84,10 @@ __vercode() { __complete_options() { case "${cur}" in --*) - COMPREPLY=( $( compgen -W "--help ${lopts}" -- $cur ) ) + COMPREPLY=( $( compgen -W "--help --version ${lopts}" -- $cur ) ) return 0;; *) - COMPREPLY=( $( compgen -W "-h ${opts} --help ${lopts}" -- $cur ) ) + COMPREPLY=( $( compgen -W "-h ${opts} --help --version ${lopts}" -- $cur ) ) return 0;; esac } diff --git a/fdroid b/fdroid index f97d7473..91dc9929 100755 --- a/fdroid +++ b/fdroid @@ -21,7 +21,7 @@ import sys import logging -from fdroidserver.common import FDroidException +import fdroidserver.common from optparse import OptionError commands = { @@ -45,7 +45,7 @@ commands = { def print_help(): - print "usage: fdroid [-h|--help] []" + print "usage: fdroid [-h|--help|--version] []" print print "Valid commands are:" for cmd, summary in commands.items(): @@ -64,6 +64,32 @@ def main(): if command in ('-h', '--help'): print_help() sys.exit(0) + elif command == '--version': + import os.path + output = 'no version info found!' + cmddir = os.path.realpath(os.path.dirname(__file__)) + moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..') + if cmddir == moduledir: + # running from git + os.chdir(cmddir) + if os.path.isdir('.git'): + import subprocess + try: + output = subprocess.check_output(['git', 'describe'], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + output = 'git commit ' + subprocess.check_output(['git', 'rev-parse', 'HEAD']) + elif os.path.exists('setup.py'): + import re + m = re.search(r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''', + open('setup.py').read(), flags=re.MULTILINE) + if m: + output = m.group(1) + '\n' + else: + from pkg_resources import get_distribution + output = get_distribution('fdroidserver').version + '\n' + print(output), + sys.exit(0) else: print "Command '%s' not recognised.\n" % command print_help() @@ -92,7 +118,7 @@ def main(): try: mod.main() # These are ours, contain a proper message and are "expected" - except FDroidException, e: + except fdroidserver.common.FDroidException, e: if verbose: raise else: diff --git a/tests/run-tests b/tests/run-tests index 65ab0d88..ce1b0fd1 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -102,6 +102,12 @@ for testcase in $WORKSPACE/tests/*.TestCase; do done +#------------------------------------------------------------------------------# +echo_header "print fdroid version" + +$fdroid --version + + #------------------------------------------------------------------------------# echo_header "build the TeX manual" From e6c0be88984225761f1271c80207e10a7ed34b60 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 1 Sep 2015 12:28:58 +0200 Subject: [PATCH 2/4] move tests into common script for jenkins and gitlab-ci --- .gitlab-ci.yml | 9 ++-- jenkins-build | 89 +---------------------------------- tests/complete-ci-tests | 101 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 92 deletions(-) create mode 100755 tests/complete-ci-tests diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 628ffe48..309695bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,9 @@ before_script: - echo " == Installing required packages" - apt-get -q install -y wget tar lib32stdc++6 lib32z1 python pyflakes pep8 dash bash ruby - python-imaging python-libcloud python-magic python-paramiko - python-pyasn1 python-pyasn1-modules python-requests - python-yaml + python-imaging python-libcloud python-logilab-astng python-magic + python-paramiko python-pip python-pyasn1 python-pyasn1-modules + python-requests python-yaml rsync - echo " == Installing OpenJDK 7" - apt-get -q install -y openjdk-7-jdk @@ -22,6 +22,5 @@ before_script: test: script: - - ./hooks/pre-commit - cd tests - - ./run-tests + - ./complete-ci-tests diff --git a/jenkins-build b/jenkins-build index 7e54312d..b7e5fd05 100755 --- a/jenkins-build +++ b/jenkins-build @@ -12,91 +12,6 @@ fi set -e set -x -if [ -z $WORKSPACE ]; then - export WORKSPACE=`pwd` -fi - -if [ -z $ANDROID_HOME ]; then - if [ -e ~/.android/bashrc ]; then - . ~/.android/bashrc - else - echo "ANDROID_HOME must be set!" - exit - fi -fi - - -#------------------------------------------------------------------------------# -# cache pypi downloads -if [ -z $PIP_DOWNLOAD_CACHE ]; then - export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache -fi - - -#------------------------------------------------------------------------------# -# required Java 7 keytool/jarsigner for :file support - -export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH - - -#------------------------------------------------------------------------------# -# run local tests, don't scan fdroidserver/ project for APKs - # this is a local repo on the Guardian Project Jenkins server -apksource=/var/www/fdroid - -cd $WORKSPACE/tests -./run-tests $apksource - - -#------------------------------------------------------------------------------# -# test building the source tarball, then installing it -cd $WORKSPACE -python2 setup.py sdist - -rm -rf $WORKSPACE/env -virtualenv --python=python2 $WORKSPACE/env -. $WORKSPACE/env/bin/activate -pip install dist/fdroidserver-*.tar.gz - -# run tests in new pip+virtualenv install -fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource - - -#------------------------------------------------------------------------------# -# test install using install direct from git repo -cd $WORKSPACE -rm -rf $WORKSPACE/env -virtualenv --python=python2 --system-site-packages $WORKSPACE/env -. $WORKSPACE/env/bin/activate -pip install -e $WORKSPACE -python2 setup.py install - -# run tests in new pip+virtualenv install -fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource - - -#------------------------------------------------------------------------------# -# run git pre-commit hook for pep8, pyflakes, etc -sh hooks/pre-commit - - -#------------------------------------------------------------------------------# -# run pylint - -cd $WORKSPACE -set +e -# use the virtualenv python so pylint checks against its installed libs - PYTHONPATH=$WORKSPACE/.pylint-plugins python2 /usr/bin/pylint \ - --output-format=parseable --reports=n \ - --load-plugins astng_hashlib \ - fdroidserver/*.py fdroid makebuildserver setup.py > $WORKSPACE/pylint.parseable - -# to only tell jenkins there was an error if we got ERROR or FATAL, uncomment these: -# running pylint in the virtualenv is causing this FATAL error, which is a bug: -# https://bitbucket.org/logilab/pylint/issue/73/pylint-is-unable-to-import -[ $(($? & 1)) = "1" ] && echo "FATALs found" -[ $(($? & 2)) = "2" ] && exit 2 -[ $(($? & 4)) = "4" ] && exit 4 -set -e - +cd tests +./complete-ci-tests /var/www/fdroid diff --git a/tests/complete-ci-tests b/tests/complete-ci-tests new file mode 100755 index 00000000..e9595b03 --- /dev/null +++ b/tests/complete-ci-tests @@ -0,0 +1,101 @@ +#!/bin/bash +# +# this is the script run by the Jenkins and gitlab-ci continuous integration +# build services. It is a thorough set of tests that runs all the tests using +# the various methods of installing/running fdroidserver. It is separate from +# ./tests/run-tests because its too heavy for manual use. + +if [ `dirname $0` != "." ]; then + echo "only run this script like ./`basename $0`" + exit +fi + +set -e +set -x + +if [ -z $WORKSPACE ]; then + export WORKSPACE=`pwd`/.. +fi + +if [ -z $ANDROID_HOME ]; then + if [ -e ~/.android/bashrc ]; then + . ~/.android/bashrc + else + echo "ANDROID_HOME must be set!" + exit + fi +fi + +apksource=$1 + +#------------------------------------------------------------------------------# +# cache pypi downloads +if [ -z $PIP_DOWNLOAD_CACHE ]; then + export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache +fi + + +#------------------------------------------------------------------------------# +# required Java 7 keytool/jarsigner for :file support + +export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH + + +#------------------------------------------------------------------------------# +# run local tests, don't scan fdroidserver/ project for APKs + +cd $WORKSPACE/tests +./run-tests $apksource + + +#------------------------------------------------------------------------------# +# test building the source tarball, then installing it +cd $WORKSPACE +python2 setup.py sdist + +rm -rf $WORKSPACE/env +virtualenv --python=python2 $WORKSPACE/env +. $WORKSPACE/env/bin/activate +pip install dist/fdroidserver-*.tar.gz + +# run tests in new pip+virtualenv install +fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource + + +#------------------------------------------------------------------------------# +# test install using install direct from git repo +cd $WORKSPACE +rm -rf $WORKSPACE/env +virtualenv --python=python2 --system-site-packages $WORKSPACE/env +. $WORKSPACE/env/bin/activate +pip install -e $WORKSPACE +python2 setup.py install + +# run tests in new pip+virtualenv install +fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource + + +#------------------------------------------------------------------------------# +# run git pre-commit hook for pep8, pyflakes, etc +sh hooks/pre-commit + + +#------------------------------------------------------------------------------# +# run pylint + +cd $WORKSPACE +set +e +# use the virtualenv python so pylint checks against its installed libs + PYTHONPATH=$WORKSPACE/.pylint-plugins python2 /usr/bin/pylint \ + --output-format=parseable --reports=n \ + --load-plugins astng_hashlib \ + fdroidserver/*.py fdroid makebuildserver setup.py > $WORKSPACE/pylint.parseable + +# to only tell jenkins there was an error if we got ERROR or FATAL, uncomment these: +# running pylint in the virtualenv is causing this FATAL error, which is a bug: +# https://bitbucket.org/logilab/pylint/issue/73/pylint-is-unable-to-import +[ $(($? & 1)) = "1" ] && echo "FATALs found" +[ $(($? & 2)) = "2" ] && exit 2 +[ $(($? & 4)) = "4" ] && exit 4 +set -e + From 242e9d2fb9992f602e4b912be813547db3d9972f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 1 Sep 2015 19:57:40 +0200 Subject: [PATCH 3/4] gitlab-ci: install all android packages at once the `android` utility is pretty stupid, it doesn't really cache the package index info. So each time it is run, it tries to fetch the indexes from the network. This combines all android package installs to a single command to make things run quicker. --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 309695bd..298bf504 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,9 +16,7 @@ before_script: - export ANDROID_HOME=$PWD/android-sdk - export PATH="$ANDROID_HOME/tools:$PATH" - echo " == Installing Android SDK components" - - echo y | android -s update sdk --no-ui -a -t platform-tools - - echo y | android -s update sdk --no-ui -a -t tools - - echo y | android -s update sdk --no-ui -a -t build-tools-23.0.0 + - echo y | android -s update sdk --no-ui -a -t platform-tools,tools,build-tools-23.0.0 test: script: From f87b17139b7838e99ca87c8a1e3c687f0a45c7d2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 1 Sep 2015 20:10:01 +0200 Subject: [PATCH 4/4] pep8: on *.TestCase, skip "E402 module level import not at top of file" The tests use a little hack in order to cleanly import the fdroidserver package locally like a regular package. pep8 doesn't see that, so this changes the pep8 to skip E402 on *.TestCase --- hooks/pre-commit | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hooks/pre-commit b/hooks/pre-commit index 891ad19b..4cc15059 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -6,7 +6,8 @@ # Redirect output to stderr. exec 1>&2 -PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py tests/*.TestCase" +PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" +PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" RB_FILES="buildserver/cookbooks/*/recipes/*.rb" @@ -48,7 +49,7 @@ else err "pep8 is not installed!" fi -if ! $PYFLAKES $PY_FILES; then +if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then err "pyflakes tests failed!" fi @@ -56,6 +57,13 @@ if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then err "pep8 tests failed!" fi +# The tests use a little hack in order to cleanly import the fdroidserver +# package locally like a regular package. pep8 doesn't see that, so this +# makes pep8 skip E402 on the test files that need that hack. +if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then + err "pep8 tests failed!" +fi + for f in $SH_FILES; do if ! dash -n $f; then