diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 628ffe48..298bf504 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,9 +3,9 @@ before_script: - echo " == Installing required packages" - apt-get -q install -y wget tar lib32stdc++6 lib32z1 python pyflakes pep8 dash bash ruby - python-imaging python-libcloud python-magic python-paramiko - python-pyasn1 python-pyasn1-modules python-requests - python-yaml + python-imaging python-libcloud python-logilab-astng python-magic + python-paramiko python-pip python-pyasn1 python-pyasn1-modules + python-requests python-yaml rsync - echo " == Installing OpenJDK 7" - apt-get -q install -y openjdk-7-jdk @@ -16,12 +16,9 @@ before_script: - export ANDROID_HOME=$PWD/android-sdk - export PATH="$ANDROID_HOME/tools:$PATH" - echo " == Installing Android SDK components" - - echo y | android -s update sdk --no-ui -a -t platform-tools - - echo y | android -s update sdk --no-ui -a -t tools - - echo y | android -s update sdk --no-ui -a -t build-tools-23.0.0 + - echo y | android -s update sdk --no-ui -a -t platform-tools,tools,build-tools-23.0.0 test: script: - - ./hooks/pre-commit - cd tests - - ./run-tests + - ./complete-ci-tests diff --git a/completion/bash-completion b/completion/bash-completion index f4dc01d8..0fd88d3b 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -84,10 +84,10 @@ __vercode() { __complete_options() { case "${cur}" in --*) - COMPREPLY=( $( compgen -W "--help ${lopts}" -- $cur ) ) + COMPREPLY=( $( compgen -W "--help --version ${lopts}" -- $cur ) ) return 0;; *) - COMPREPLY=( $( compgen -W "-h ${opts} --help ${lopts}" -- $cur ) ) + COMPREPLY=( $( compgen -W "-h ${opts} --help --version ${lopts}" -- $cur ) ) return 0;; esac } diff --git a/fdroid b/fdroid index f97d7473..91dc9929 100755 --- a/fdroid +++ b/fdroid @@ -21,7 +21,7 @@ import sys import logging -from fdroidserver.common import FDroidException +import fdroidserver.common from optparse import OptionError commands = { @@ -45,7 +45,7 @@ commands = { def print_help(): - print "usage: fdroid [-h|--help] []" + print "usage: fdroid [-h|--help|--version] []" print print "Valid commands are:" for cmd, summary in commands.items(): @@ -64,6 +64,32 @@ def main(): if command in ('-h', '--help'): print_help() sys.exit(0) + elif command == '--version': + import os.path + output = 'no version info found!' + cmddir = os.path.realpath(os.path.dirname(__file__)) + moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..') + if cmddir == moduledir: + # running from git + os.chdir(cmddir) + if os.path.isdir('.git'): + import subprocess + try: + output = subprocess.check_output(['git', 'describe'], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + output = 'git commit ' + subprocess.check_output(['git', 'rev-parse', 'HEAD']) + elif os.path.exists('setup.py'): + import re + m = re.search(r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''', + open('setup.py').read(), flags=re.MULTILINE) + if m: + output = m.group(1) + '\n' + else: + from pkg_resources import get_distribution + output = get_distribution('fdroidserver').version + '\n' + print(output), + sys.exit(0) else: print "Command '%s' not recognised.\n" % command print_help() @@ -92,7 +118,7 @@ def main(): try: mod.main() # These are ours, contain a proper message and are "expected" - except FDroidException, e: + except fdroidserver.common.FDroidException, e: if verbose: raise else: diff --git a/hooks/pre-commit b/hooks/pre-commit index 891ad19b..4cc15059 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -6,7 +6,8 @@ # Redirect output to stderr. exec 1>&2 -PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py tests/*.TestCase" +PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" +PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion" RB_FILES="buildserver/cookbooks/*/recipes/*.rb" @@ -48,7 +49,7 @@ else err "pep8 is not installed!" fi -if ! $PYFLAKES $PY_FILES; then +if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then err "pyflakes tests failed!" fi @@ -56,6 +57,13 @@ if ! $PEP8 --ignore=$PEP8_IGNORE $PY_FILES; then err "pep8 tests failed!" fi +# The tests use a little hack in order to cleanly import the fdroidserver +# package locally like a regular package. pep8 doesn't see that, so this +# makes pep8 skip E402 on the test files that need that hack. +if ! $PEP8 --ignore=$PEP8_IGNORE,E402 $PY_TEST_FILES; then + err "pep8 tests failed!" +fi + for f in $SH_FILES; do if ! dash -n $f; then diff --git a/jenkins-build b/jenkins-build index 7e54312d..b7e5fd05 100755 --- a/jenkins-build +++ b/jenkins-build @@ -12,91 +12,6 @@ fi set -e set -x -if [ -z $WORKSPACE ]; then - export WORKSPACE=`pwd` -fi - -if [ -z $ANDROID_HOME ]; then - if [ -e ~/.android/bashrc ]; then - . ~/.android/bashrc - else - echo "ANDROID_HOME must be set!" - exit - fi -fi - - -#------------------------------------------------------------------------------# -# cache pypi downloads -if [ -z $PIP_DOWNLOAD_CACHE ]; then - export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache -fi - - -#------------------------------------------------------------------------------# -# required Java 7 keytool/jarsigner for :file support - -export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH - - -#------------------------------------------------------------------------------# -# run local tests, don't scan fdroidserver/ project for APKs - # this is a local repo on the Guardian Project Jenkins server -apksource=/var/www/fdroid - -cd $WORKSPACE/tests -./run-tests $apksource - - -#------------------------------------------------------------------------------# -# test building the source tarball, then installing it -cd $WORKSPACE -python2 setup.py sdist - -rm -rf $WORKSPACE/env -virtualenv --python=python2 $WORKSPACE/env -. $WORKSPACE/env/bin/activate -pip install dist/fdroidserver-*.tar.gz - -# run tests in new pip+virtualenv install -fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource - - -#------------------------------------------------------------------------------# -# test install using install direct from git repo -cd $WORKSPACE -rm -rf $WORKSPACE/env -virtualenv --python=python2 --system-site-packages $WORKSPACE/env -. $WORKSPACE/env/bin/activate -pip install -e $WORKSPACE -python2 setup.py install - -# run tests in new pip+virtualenv install -fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource - - -#------------------------------------------------------------------------------# -# run git pre-commit hook for pep8, pyflakes, etc -sh hooks/pre-commit - - -#------------------------------------------------------------------------------# -# run pylint - -cd $WORKSPACE -set +e -# use the virtualenv python so pylint checks against its installed libs - PYTHONPATH=$WORKSPACE/.pylint-plugins python2 /usr/bin/pylint \ - --output-format=parseable --reports=n \ - --load-plugins astng_hashlib \ - fdroidserver/*.py fdroid makebuildserver setup.py > $WORKSPACE/pylint.parseable - -# to only tell jenkins there was an error if we got ERROR or FATAL, uncomment these: -# running pylint in the virtualenv is causing this FATAL error, which is a bug: -# https://bitbucket.org/logilab/pylint/issue/73/pylint-is-unable-to-import -[ $(($? & 1)) = "1" ] && echo "FATALs found" -[ $(($? & 2)) = "2" ] && exit 2 -[ $(($? & 4)) = "4" ] && exit 4 -set -e - +cd tests +./complete-ci-tests /var/www/fdroid diff --git a/tests/complete-ci-tests b/tests/complete-ci-tests new file mode 100755 index 00000000..e9595b03 --- /dev/null +++ b/tests/complete-ci-tests @@ -0,0 +1,101 @@ +#!/bin/bash +# +# this is the script run by the Jenkins and gitlab-ci continuous integration +# build services. It is a thorough set of tests that runs all the tests using +# the various methods of installing/running fdroidserver. It is separate from +# ./tests/run-tests because its too heavy for manual use. + +if [ `dirname $0` != "." ]; then + echo "only run this script like ./`basename $0`" + exit +fi + +set -e +set -x + +if [ -z $WORKSPACE ]; then + export WORKSPACE=`pwd`/.. +fi + +if [ -z $ANDROID_HOME ]; then + if [ -e ~/.android/bashrc ]; then + . ~/.android/bashrc + else + echo "ANDROID_HOME must be set!" + exit + fi +fi + +apksource=$1 + +#------------------------------------------------------------------------------# +# cache pypi downloads +if [ -z $PIP_DOWNLOAD_CACHE ]; then + export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache +fi + + +#------------------------------------------------------------------------------# +# required Java 7 keytool/jarsigner for :file support + +export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH + + +#------------------------------------------------------------------------------# +# run local tests, don't scan fdroidserver/ project for APKs + +cd $WORKSPACE/tests +./run-tests $apksource + + +#------------------------------------------------------------------------------# +# test building the source tarball, then installing it +cd $WORKSPACE +python2 setup.py sdist + +rm -rf $WORKSPACE/env +virtualenv --python=python2 $WORKSPACE/env +. $WORKSPACE/env/bin/activate +pip install dist/fdroidserver-*.tar.gz + +# run tests in new pip+virtualenv install +fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource + + +#------------------------------------------------------------------------------# +# test install using install direct from git repo +cd $WORKSPACE +rm -rf $WORKSPACE/env +virtualenv --python=python2 --system-site-packages $WORKSPACE/env +. $WORKSPACE/env/bin/activate +pip install -e $WORKSPACE +python2 setup.py install + +# run tests in new pip+virtualenv install +fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource + + +#------------------------------------------------------------------------------# +# run git pre-commit hook for pep8, pyflakes, etc +sh hooks/pre-commit + + +#------------------------------------------------------------------------------# +# run pylint + +cd $WORKSPACE +set +e +# use the virtualenv python so pylint checks against its installed libs + PYTHONPATH=$WORKSPACE/.pylint-plugins python2 /usr/bin/pylint \ + --output-format=parseable --reports=n \ + --load-plugins astng_hashlib \ + fdroidserver/*.py fdroid makebuildserver setup.py > $WORKSPACE/pylint.parseable + +# to only tell jenkins there was an error if we got ERROR or FATAL, uncomment these: +# running pylint in the virtualenv is causing this FATAL error, which is a bug: +# https://bitbucket.org/logilab/pylint/issue/73/pylint-is-unable-to-import +[ $(($? & 1)) = "1" ] && echo "FATALs found" +[ $(($? & 2)) = "2" ] && exit 2 +[ $(($? & 4)) = "4" ] && exit 4 +set -e + diff --git a/tests/run-tests b/tests/run-tests index 65ab0d88..ce1b0fd1 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -102,6 +102,12 @@ for testcase in $WORKSPACE/tests/*.TestCase; do done +#------------------------------------------------------------------------------# +echo_header "print fdroid version" + +$fdroid --version + + #------------------------------------------------------------------------------# echo_header "build the TeX manual"