diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3049262f..2dbea825 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,7 @@ metadata_v0: - git reset --hard - git checkout $GITCOMMIT - cd fdroiddata + - echo "accepted_formats = ('txt', 'yml')" >> config.py - ../tests/dump_internal_metadata_format.py - sed -i -e '/Translation:/d' @@ -79,6 +80,28 @@ ubuntu_lts: - cd tests - ./run-tests +# test using TrustyLTS with all depends from pypi +ubuntu_trusty_pip: + image: ubuntu:trusty + only: + - master@fdroid/fdroidserver + variables: + DEBIAN_FRONTEND: noninteractive + LANG: C.UTF-8 + script: + - echo Etc/UTC > /etc/timezone + - apt-get -qy update + - apt-get -qy dist-upgrade + - apt-get -qy install git default-jdk python3-pip python3.4-venv + - rm -rf env + - pyvenv-3.4 env + - . env/bin/activate + - echo sed -i "s/'requests.*',$/'requests',/" setup.py + - pip3 install --upgrade babel pip setuptools + - pip3 install -e . + - ./setup.py compile_catalog + - ./tests/run-tests + pip_install: image: archlinux/base only: @@ -98,6 +121,30 @@ pip_install: - fdroid readmeta - fdroid update --help +pyup_io_safety_check: + image: archlinux/base + script: + - pacman --sync --sysupgrade --refresh --noconfirm grep python-pip python-virtualenv tar + - ./setup.py compile_catalog install + - pip install safety + - safety check --full-report + +pylint: + image: alpine:3.7 + variables: + LANG: C.UTF-8 + script: + - apk add --no-cache ca-certificates python3 + - python3 -m ensurepip + - pip3 install pylint + - pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n + fdroid + makebuildserver + setup.py + fdroidserver/*.py + tests/*.py + tests/*.TestCase + fedora_latest: image: fedora:latest only: diff --git a/.travis.yml b/.travis.yml index 952eda78..1c887fed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,27 +24,18 @@ matrix: # is used on Windows Subsystem for Linux. addons: apt: + update: true sources: - sourceline: 'ppa:fdroid/fdroidserver' packages: - - bash - - dash - - pylint - - pep8 - - python3-babel - - python3-dev - - python3-pip - - python3-ruamel.yaml - - python3-setuptools - - python3.4-venv - - libjpeg-dev - - zlib1g-dev - - fdroidserver + - python3-babel + - python3-setuptools + - fdroidserver android: components: - android-23 # required for `fdroid build` test - - build-tools-25.0.3 # required for `fdroid build` test + - build-tools-27.0.3 # required for `fdroid build` test licenses: - 'android-sdk-preview-.+' - 'android-sdk-license-.+' @@ -101,15 +92,12 @@ install: fi # The OSX tests seem to run slower, they often timeout. So only run -# the test suite with the installed version of fdroid, instead of the -# three rounds that ./complete-ci-tests does. +# the test suite with the installed version of fdroid. +# +# Supporting pip on Ubuntu/trusty was too painful here, since it seems +# that pip installs conflict with the Ubuntu packages. script: - - cd tests - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - ./run-tests; - else - ./complete-ci-tests; - fi + - ./tests/run-tests after_failure: - cd $TRAVIS_BUILD_DIR diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 56caaad1..29bd24cb 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -163,7 +163,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ output['requests'] = requestsdict # establish sort order of the index - v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints) + v1_sort_packages(packages, fdroid_signing_key_fingerprints) appslist = [] output['apps'] = appslist @@ -252,7 +252,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ signindex.sign_index_v1(repodir, json_name) -def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints): +def v1_sort_packages(packages, fdroid_signing_key_fingerprints): """Sorts the supplied list to ensure a deterministic sort order for package entries in the index file. This sort-order also expresses installation preference to the clients. diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index d50bfaa1..c0dd55d3 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -52,7 +52,7 @@ def publish_source_tarball(apkfilename, unsigned_dir, output_dir): logging.debug('...no source tarball for %s', apkfilename) -def key_alias(appid, resolve=False): +def key_alias(appid): """Get the alias which F-Droid uses to indentify the singing key for this App in F-Droids keystore. """ diff --git a/fdroidserver/signatures.py b/fdroidserver/signatures.py index 7d82ad14..e104b483 100644 --- a/fdroidserver/signatures.py +++ b/fdroidserver/signatures.py @@ -45,7 +45,7 @@ def extract_signature(apkpath): return sigdir -def extract(config, options): +def extract(options): # Create tmp dir if missing… tmp_dir = 'tmp' diff --git a/jenkins-setup-build-environment b/jenkins-setup-build-environment index dc9a7751..cab721c6 100755 --- a/jenkins-setup-build-environment +++ b/jenkins-setup-build-environment @@ -107,5 +107,3 @@ fi ../fdroid build --verbose --stop org.adaway:55 # building old versions should still work ../fdroid build --verbose --stop org.fdroid.fdroid:96150 -# test OTA update ZIP build and publish -../fdroid build --verbose --stop org.fdroid.fdroid.privileged.ota:2070 diff --git a/setup.py b/setup.py index f50c4d86..e9ef1b7e 100755 --- a/setup.py +++ b/setup.py @@ -82,14 +82,15 @@ setup(name='fdroidserver', 'babel', ], install_requires=[ + 'androguard >= 3.1.0rc2', 'clint', 'GitPython', 'mwclient', 'paramiko', 'Pillow', 'apache-libcloud >= 0.14.1', - 'pyasn1', - 'pyasn1-modules', + 'pyasn1 <0.5.0, >=0.4.1', + 'pyasn1-modules == 0.2.1', 'python-vagrant', 'PyYAML', 'qrcode', diff --git a/tests/complete-ci-tests b/tests/complete-ci-tests index 6201eb2c..75fe515e 100755 --- a/tests/complete-ci-tests +++ b/tests/complete-ci-tests @@ -27,6 +27,11 @@ if [ -z $ANDROID_HOME ]; then fi fi +if ! which pyvenv; then + echo "pyvenv required to run this test suite!" + exit 1 +fi + apksource=$1 #------------------------------------------------------------------------------# @@ -35,31 +40,12 @@ if [ -z $PIP_DOWNLOAD_CACHE ]; then export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache fi - -#------------------------------------------------------------------------------# -# required Java 7 or later keytool/jarsigner for :file support - -export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin:$PATH - - #------------------------------------------------------------------------------# # run local tests, don't scan fdroidserver/ project for APKs cd $WORKSPACE/tests ./run-tests $apksource -#------------------------------------------------------------------------------# -# find pyvenv, to support Ubuntu/trusty's python3.4-venv - -if which pyvenv; then - pyvenv=pyvenv -elif which pyvenv-3.4; then - pyvenv=pyvenv-3.4 -else - echo "pyvenv required to run this test suite!" - exit 1 -fi - #------------------------------------------------------------------------------# # make sure that translations do not cause stacktraces @@ -78,10 +64,8 @@ done # test install using install direct from git repo cd $WORKSPACE rm -rf $WORKSPACE/env -$pyvenv $WORKSPACE/env +pyvenv $WORKSPACE/env . $WORKSPACE/env/bin/activate -# workaround https://github.com/pypa/setuptools/issues/937 -pip3 install --quiet setuptools==33.1.1 Babel pip3 install --quiet -e $WORKSPACE python3 setup.py compile_catalog install @@ -90,19 +74,3 @@ test -e $WORKSPACE/env/share/locale/de/LC_MESSAGES/fdroidserver.mo # run tests in new pip+pyvenv install fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource - - -#------------------------------------------------------------------------------# -# run pylint - -# only run it where it will work, for example, the pyvenvs above don't have pylint -if which pylint3 && python3 -c "import pylint" 2> /dev/null; then - cd $WORKSPACE - pylint3 --rcfile=.pylint-rcfile --output-format=colorized --reports=n \ - fdroid \ - makebuildserver \ - setup.py \ - fdroidserver/*.py \ - tests/*.py \ - tests/*.TestCase -fi diff --git a/tests/index.TestCase b/tests/index.TestCase index 1fd89f7e..15fc4deb 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -211,7 +211,7 @@ class IndexTest(unittest.TestCase): pass fdroidserver.index.v1_sort_packages( - i, 'repo', fdroidserver.common.load_stats_fdroid_signing_key_fingerprints()) + i, fdroidserver.common.load_stats_fdroid_signing_key_fingerprints()) self.maxDiff = None self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2)) diff --git a/tests/run-tests b/tests/run-tests index 4d35d8bf..df03a2e2 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -156,31 +156,32 @@ $fdroid --version #------------------------------------------------------------------------------# echo_header 'run process when building and signing are on separate machines' -REPOROOT=`create_test_dir` -cd $REPOROOT -cp $WORKSPACE/tests/keystore.jks $REPOROOT/ -$fdroid init --keystore keystore.jks --repo-keyalias=sova -echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py -echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py -echo "accepted_formats = ['txt', 'yml']" >> config.py -echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py -test -d archive || mkdir archive -test -d metadata || mkdir metadata -cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ -test -d repo || mkdir repo -test -d unsigned || mkdir unsigned -cp $WORKSPACE/tests/urzip-release-unsigned.apk unsigned/info.guardianproject.urzip_100.apk -$fdroid publish --verbose -$fdroid update --verbose --nosign -$fdroid signindex --verbose -test -e repo/index.xml -test -e repo/index.jar -test -e repo/index-v1.jar -test -e tmp/apkcache -! test -z tmp/apkcache -test -L urzip.apk -grep -F '> config.py + echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py + echo "accepted_formats = ['txt', 'yml']" >> config.py + echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py + test -d archive || mkdir archive + test -d metadata || mkdir metadata + cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ + test -d repo || mkdir repo + test -d unsigned || mkdir unsigned + cp $WORKSPACE/tests/urzip-release-unsigned.apk unsigned/info.guardianproject.urzip_100.apk + $fdroid publish --verbose + $fdroid update --verbose --nosign + $fdroid signindex --verbose + test -e repo/index.xml + test -e repo/index.jar + test -e repo/index-v1.jar + test -e tmp/apkcache + ! test -z tmp/apkcache + test -L urzip.apk + grep -F '> config.py + echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py + echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py + echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py + $fdroid update --pretty + grep -F '' repo/index.xml + grep -F '/fdroid/archive' archive/index.xml + test `grep '' repo/index.xml | wc -l` -eq 2 + test `grep '' archive/index.xml | wc -l` -eq 2 + cd binary_transparency + [ `git rev-list --count HEAD` == "1" ] + cd .. + $fdroid server update --verbose + grep -F '> config.py + echo "sync_from_local_copy_dir = True" >> config.py + echo "serverwebroots = '$SERVERWEBROOT'" >> config.py + echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py + echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py + echo "binary_transparency_remote = '$BINARY_TRANSPARENCY_REMOTE'" >> config.py + $fdroid server update --verbose + cd $BINARY_TRANSPARENCY_REMOTE + [ `git rev-list --count HEAD` == "1" ] + cd $SERVER_GIT_MIRROR + [ `git rev-list --count HEAD` == "1" ] fi -cd $OFFLINE_ROOT -fdroid_init_with_prebuilt_keystore -cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $OFFLINE_ROOT/ - -echo "mirrors = ['http://foo.bar/fdroid', 'http://asdflkdsfjafdsdfhkjh.onion/fdroid']" >> config.py -echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py -echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py -echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py -$fdroid update --pretty -grep -F '' repo/index.xml -grep -F '/fdroid/archive' archive/index.xml -test `grep '' repo/index.xml | wc -l` -eq 2 -test `grep '' archive/index.xml | wc -l` -eq 2 -cd binary_transparency -[ `git rev-list --count HEAD` == "1" ] -cd .. -$fdroid server update --verbose -grep -F '> config.py -echo "sync_from_local_copy_dir = True" >> config.py -echo "serverwebroots = '$SERVERWEBROOT'" >> config.py -echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py -echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py -echo "binary_transparency_remote = '$BINARY_TRANSPARENCY_REMOTE'" >> config.py -$fdroid server update --verbose -cd $BINARY_TRANSPARENCY_REMOTE -[ `git rev-list --count HEAD` == "1" ] -cd $SERVER_GIT_MIRROR -[ `git rev-list --count HEAD` == "1" ] - #------------------------------------------------------------------------------# diff --git a/tests/signatures.TestCase b/tests/signatures.TestCase index bd6648cd..276f9144 100755 --- a/tests/signatures.TestCase +++ b/tests/signatures.TestCase @@ -36,7 +36,7 @@ class SignaturesTest(unittest.TestCase): APK = [os.path.abspath(os.path.join('repo', 'com.politedroid_3.apk'))] with TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): - signatures.extract(common.config, OptionsFixture()) + signatures.extract(OptionsFixture()) # check if extracted signatures are where they are supposed to be # also verify weather if extracted file contian what they should