Merge branch 'fdroid-version' into 'master'

fdroid --version and shared tests between gitlab and jenkins

This adds a `fdroid --version` flag for people to easily see the exact version of fdroidserver in use.  It'll also report the version using `git describe` when running from git.

The other moves the extended tests out of `./jenkins-build` into a common script for both gitlab-ci and jenkins.

See merge request !70

.gitlab-ci.yml

@@ -3,9 +3,9 @@ before_script:
   - echo " == Installing required packages"
   - apt-get -q install -y wget tar lib32stdc++6 lib32z1
       python pyflakes pep8 dash bash ruby
-      python-imaging python-libcloud python-magic python-paramiko
+      python-imaging python-libcloud python-logilab-astng python-magic
-      python-pyasn1 python-pyasn1-modules python-requests
+      python-paramiko python-pip python-pyasn1 python-pyasn1-modules
-      python-yaml
+      python-requests python-yaml
       rsync
   - echo " == Installing OpenJDK 7"
   - apt-get -q install -y openjdk-7-jdk
@@ -16,12 +16,9 @@ before_script:
   - export ANDROID_HOME=$PWD/android-sdk
   - export PATH="$ANDROID_HOME/tools:$PATH"
   - echo " == Installing Android SDK components"
-  - echo y | android -s update sdk --no-ui -a -t platform-tools
+  - echo y | android -s update sdk --no-ui -a -t platform-tools,tools,build-tools-23.0.0
-  - echo y | android -s update sdk --no-ui -a -t tools
-  - echo y | android -s update sdk --no-ui -a -t build-tools-23.0.0
 
 test:
   script:
-    - ./hooks/pre-commit
     - cd tests
-    - ./run-tests
+    - ./complete-ci-tests

completion/bash-completion

@@ -84,10 +84,10 @@ __vercode() {
 __complete_options() {
 	case "${cur}" in
 		--*)
-			COMPREPLY=( $( compgen -W "--help ${lopts}" -- $cur ) )
+			COMPREPLY=( $( compgen -W "--help --version ${lopts}" -- $cur ) )
 			return 0;;
 		*)
-			COMPREPLY=( $( compgen -W "-h ${opts} --help ${lopts}" -- $cur ) )
+			COMPREPLY=( $( compgen -W "-h ${opts} --help --version ${lopts}" -- $cur ) )
 			return 0;;
 	esac
 }

fdroid

@@ -21,7 +21,7 @@
 import sys
 import logging
 
-from fdroidserver.common import FDroidException
+import fdroidserver.common
 from optparse import OptionError
 
 commands = {
@@ -45,7 +45,7 @@ commands = {
 
 
 def print_help():
-    print "usage: fdroid [-h|--help] <command> [<args>]"
+    print "usage: fdroid [-h|--help|--version] <command> [<args>]"
     print
     print "Valid commands are:"
     for cmd, summary in commands.items():
@@ -64,6 +64,32 @@ def main():
         if command in ('-h', '--help'):
             print_help()
             sys.exit(0)
+        elif command == '--version':
+            import os.path
+            output = 'no version info found!'
+            cmddir = os.path.realpath(os.path.dirname(__file__))
+            moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..')
+            if cmddir == moduledir:
+                # running from git
+                os.chdir(cmddir)
+                if os.path.isdir('.git'):
+                    import subprocess
+                    try:
+                        output = subprocess.check_output(['git', 'describe'],
+                                                         stderr=subprocess.STDOUT)
+                    except subprocess.CalledProcessError:
+                        output = 'git commit ' + subprocess.check_output(['git', 'rev-parse', 'HEAD'])
+                elif os.path.exists('setup.py'):
+                    import re
+                    m = re.search(r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''',
+                                  open('setup.py').read(), flags=re.MULTILINE)
+                    if m:
+                        output = m.group(1) + '\n'
+            else:
+                from pkg_resources import get_distribution
+                output = get_distribution('fdroidserver').version + '\n'
+            print(output),
+            sys.exit(0)
         else:
             print "Command '%s' not recognised.\n" % command
             print_help()
@@ -92,7 +118,7 @@ def main():
     try:
         mod.main()
     # These are ours, contain a proper message and are "expected"
-    except FDroidException, e:
+    except fdroidserver.common.FDroidException, e:
         if verbose:
             raise
         else:

hooks/pre-commit

@@ -6,7 +6,8 @@
 # Redirect output to stderr.
 exec 1>&2
 
-PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py tests/*.TestCase"
+PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py"
+PY_TEST_FILES="tests/*.TestCase"
 SH_FILES="hooks/pre-commit"
 BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion"
 RB_FILES="buildserver/cookbooks/*/recipes/*.rb"
@@ -48,7 +49,7 @@ else
 	err "pep8 is not installed!"
 fi
 
-if ! $PYFLAKES $PY_FILES; then
+if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then
 	err "pyflakes tests failed!"
 fi
 
@@ -56,6 +57,13 @@ if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then
 	err "pep8 tests failed!"
 fi
 
+# The tests use a little hack in order to cleanly import the fdroidserver
+# package locally like a regular package.  pep8 doesn't see that, so this
+# makes pep8 skip E402 on the test files that need that hack.
+if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then
+	err "pep8 tests failed!"
+fi
+
 
 for f in $SH_FILES; do
 	if ! dash -n $f; then

jenkins-build

@@ -12,91 +12,6 @@ fi
 set -e
 set -x
 
-if [ -z $WORKSPACE ]; then
-    export WORKSPACE=`pwd`
-fi
-
-if [ -z $ANDROID_HOME ]; then
-    if [ -e ~/.android/bashrc ]; then
-        . ~/.android/bashrc
-    else
-        echo "ANDROID_HOME must be set!"
-        exit
-    fi
-fi
-
-
-#------------------------------------------------------------------------------#
-# cache pypi downloads
-if [ -z $PIP_DOWNLOAD_CACHE ]; then
-    export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache
-fi
-
-
-#------------------------------------------------------------------------------#
-# required Java 7 keytool/jarsigner for :file support
-
-export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
-
-
-#------------------------------------------------------------------------------#
-# run local tests, don't scan fdroidserver/ project for APKs
-
 # this is a local repo on the Guardian Project Jenkins server
-apksource=/var/www/fdroid
+cd tests
-
+./complete-ci-tests /var/www/fdroid
-cd $WORKSPACE/tests
-./run-tests $apksource
-
-
-#------------------------------------------------------------------------------#
-# test building the source tarball, then installing it
-cd $WORKSPACE
-python2 setup.py sdist
-
-rm -rf $WORKSPACE/env
-virtualenv --python=python2 $WORKSPACE/env
-. $WORKSPACE/env/bin/activate
-pip install dist/fdroidserver-*.tar.gz
-
-# run tests in new pip+virtualenv install
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
-
-
-#------------------------------------------------------------------------------#
-# test install using install direct from git repo
-cd $WORKSPACE
-rm -rf $WORKSPACE/env
-virtualenv --python=python2 --system-site-packages $WORKSPACE/env
-. $WORKSPACE/env/bin/activate
-pip install -e $WORKSPACE
-python2 setup.py install
-
-# run tests in new pip+virtualenv install
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
-
-
-#------------------------------------------------------------------------------#
-# run git pre-commit hook for pep8, pyflakes, etc
-sh hooks/pre-commit
-
-
-#------------------------------------------------------------------------------#
-# run pylint
-
-cd $WORKSPACE
-set +e
-# use the virtualenv python so pylint checks against its installed libs
-    PYTHONPATH=$WORKSPACE/.pylint-plugins python2 /usr/bin/pylint \
-        --output-format=parseable --reports=n \
-        --load-plugins astng_hashlib \
-        fdroidserver/*.py fdroid makebuildserver setup.py > $WORKSPACE/pylint.parseable
-
-# to only tell jenkins there was an error if we got ERROR or FATAL, uncomment these:
-# running pylint in the virtualenv is causing this FATAL error, which is a bug:
-# https://bitbucket.org/logilab/pylint/issue/73/pylint-is-unable-to-import
-[ $(($? & 1)) = "1" ] && echo "FATALs found"
-[ $(($? & 2)) = "2" ] && exit 2
-[ $(($? & 4)) = "4" ] && exit 4
-set -e
-

tests/complete-ci-tests

@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# this is the script run by the Jenkins and gitlab-ci continuous integration
+# build services.  It is a thorough set of tests that runs all the tests using
+# the various methods of installing/running fdroidserver.  It is separate from
+# ./tests/run-tests because its too heavy for manual use.
+
+if [ `dirname $0` != "." ]; then
+    echo "only run this script like ./`basename $0`"
+    exit
+fi
+
+set -e
+set -x
+
+if [ -z $WORKSPACE ]; then
+    export WORKSPACE=`pwd`/..
+fi
+
+if [ -z $ANDROID_HOME ]; then
+    if [ -e ~/.android/bashrc ]; then
+        . ~/.android/bashrc
+    else
+        echo "ANDROID_HOME must be set!"
+        exit
+    fi
+fi
+
+apksource=$1
+
+#------------------------------------------------------------------------------#
+# cache pypi downloads
+if [ -z $PIP_DOWNLOAD_CACHE ]; then
+    export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache
+fi
+
+
+#------------------------------------------------------------------------------#
+# required Java 7 keytool/jarsigner for :file support
+
+export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
+
+
+#------------------------------------------------------------------------------#
+# run local tests, don't scan fdroidserver/ project for APKs
+
+cd $WORKSPACE/tests
+./run-tests $apksource
+
+
+#------------------------------------------------------------------------------#
+# test building the source tarball, then installing it
+cd $WORKSPACE
+python2 setup.py sdist
+
+rm -rf $WORKSPACE/env
+virtualenv --python=python2 $WORKSPACE/env
+. $WORKSPACE/env/bin/activate
+pip install dist/fdroidserver-*.tar.gz
+
+# run tests in new pip+virtualenv install
+fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
+
+
+#------------------------------------------------------------------------------#
+# test install using install direct from git repo
+cd $WORKSPACE
+rm -rf $WORKSPACE/env
+virtualenv --python=python2 --system-site-packages $WORKSPACE/env
+. $WORKSPACE/env/bin/activate
+pip install -e $WORKSPACE
+python2 setup.py install
+
+# run tests in new pip+virtualenv install
+fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
+
+
+#------------------------------------------------------------------------------#
+# run git pre-commit hook for pep8, pyflakes, etc
+sh hooks/pre-commit
+
+
+#------------------------------------------------------------------------------#
+# run pylint
+
+cd $WORKSPACE
+set +e
+# use the virtualenv python so pylint checks against its installed libs
+    PYTHONPATH=$WORKSPACE/.pylint-plugins python2 /usr/bin/pylint \
+        --output-format=parseable --reports=n \
+        --load-plugins astng_hashlib \
+        fdroidserver/*.py fdroid makebuildserver setup.py > $WORKSPACE/pylint.parseable
+
+# to only tell jenkins there was an error if we got ERROR or FATAL, uncomment these:
+# running pylint in the virtualenv is causing this FATAL error, which is a bug:
+# https://bitbucket.org/logilab/pylint/issue/73/pylint-is-unable-to-import
+[ $(($? & 1)) = "1" ] && echo "FATALs found"
+[ $(($? & 2)) = "2" ] && exit 2
+[ $(($? & 4)) = "4" ] && exit 4
+set -e
+

tests/run-tests

@@ -102,6 +102,12 @@ for testcase in $WORKSPACE/tests/*.TestCase; do
 done
 
 
+#------------------------------------------------------------------------------#
+echo_header "print fdroid version"
+
+$fdroid --version
+
+
 #------------------------------------------------------------------------------#
 echo_header "build the TeX manual"