Merge branch 'gitlab-ci-testing-xenial-fedora' into 'master'

gitlab CI runs on  Debian/testing,  Ubuntu/xenial,  Fedora

See merge request fdroid/fdroidserver!398

.gitlab-ci.yml

@@ -1,6 +1,6 @@
-image: registry.gitlab.com/fdroid/ci-images-server:latest
 
 test:
+  image: registry.gitlab.com/fdroid/ci-images-server:latest
   script:
     - pip3 install -e .
     - cd tests
@@ -10,6 +10,7 @@ test:
 # released version. Ensure that the official tags are included when
 # running these tests on forks as well.
 metadata_v0:
+  image: registry.gitlab.com/fdroid/ci-images-server:latest
   script:
     - git fetch https://gitlab.com/fdroid/fdroidserver 0.8
     - cd tests
@@ -25,3 +26,92 @@ metadata_v0:
     - cd fdroiddata
     - ../tests/dump_internal_metadata_format.py
     - diff -uw metadata/dump_*
+
+debian_testing:
+  image: debian:testing
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - apt update -y
+    - apt dist-upgrade -y
+    - apt-get install -y --no-install-recommends
+        aapt adb android-platform-tools-base android-sdk-common fdroidserver
+        git gnupg python3-setuptools zipalign
+    - export ANDROID_HOME=/usr/lib/android-sdk
+    - export LANG=C.UTF-8
+    - cd tests
+    - rm -f install.TestCase  # fails frequently and is unimportant
+    - ./run-tests
+
+ubuntu_lts:
+  image: ubuntu:latest
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151
+    - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
+    - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
+    - apt update -y
+    - apt dist-upgrade -y
+    - apt-get install -y --no-install-recommends
+        aapt adb android-platform-tools-base android-sdk-common fdroidserver
+        git gnupg python3-setuptools unzip wget zipalign
+    - export ANDROID_HOME=/usr/lib/android-sdk
+    # xenial's aapt is too old
+    - wget --no-verbose https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip
+    - unzip -q build-tools_r27.0.1-linux.zip
+    - rm build-tools_r27.0.1-linux.zip
+    - mv android-8.1.0 $ANDROID_HOME/build-tools/27.0.1
+    - export LANG=C.UTF-8
+    - cd tests
+    - ./run-tests
+
+pip_install:
+  image: archlinux/base
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - pacman --sync --sysupgrade --refresh --noconfirm grep python-pip python-virtualenv tar
+    # setup venv to act as release build machine
+    - python -m venv sdist-env
+    - . sdist-env/bin/activate
+    - ./setup.py compile_catalog sdist
+    - deactivate
+    - tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo
+    # back to bare machine to act as user's install machine
+    - pip install dist/fdroidserver-*.tar.gz
+    - test -e /usr/share/locale/de/LC_MESSAGES/fdroidserver.mo
+    - fdroid
+    - fdroid readmeta
+    - fdroid update --help
+
+fedora_latest:
+  image: fedora:latest
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - dnf -y update
+    - dnf -y install git gnupg java-1.8.0-openjdk-devel python3 python3-babel
+                     python3-pip rsync unzip wget
+    - ./setup.py compile_catalog sdist
+    - useradd -m -c "test account" --password "fakepassword"  testuser
+    - su testuser --login --command "cd `pwd`; pip3 install --user dist/fdroidserver-*.tar.gz"
+    - test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
+    - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.4-linux.zip
+    - unzip -q tools.zip
+    - rm tools.zip
+    - export ANDROID_HOME=`pwd`/android-sdk
+    - mkdir $ANDROID_HOME
+    - mv tools $ANDROID_HOME/
+    - mkdir -p $ANDROID_HOME/licenses/
+    - printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e" > $ANDROID_HOME/licenses/android-sdk-license
+    - printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
+    - printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old
+    - mkdir ~/.android
+    - touch ~/.android/repositories.cfg
+    - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools"
+    - echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;25.0.2"
+    - chown -R testuser .
+    - cd tests
+    - su testuser --login --command
+        "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"

fdroidserver/build.py

@@ -100,6 +100,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         # Helper to copy the contents of a directory to the server...
         def send_dir(path):
             logging.debug("rsyncing " + path + " to " + ftp.getcwd())
+            # TODO this should move to `vagrant rsync` from >= v1.5
             try:
                 subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
                                          'ssh -o StrictHostKeyChecking=no' +

fdroidserver/common.py

@@ -388,8 +388,14 @@ def test_aapt_version(aapt):
             minor = m.group(2)
             bugfix = m.group(3)
             # the Debian package has the version string like "v0.2-23.0.2"
-            if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
+            too_old = False
-                logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!")
+            if '.' in bugfix:
+                if LooseVersion(bugfix) < LooseVersion('24.0.0'):
+                    too_old = True
+            elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2964546'):
+                too_old = True
+            if too_old:
+                logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-24.0.0 or newer!")
                                 .format(aapt=aapt))
         else:
             logging.warning(_('Unknown version of aapt, might cause problems: ') + output)

fdroidserver/init.py

@@ -144,7 +144,7 @@ def main():
             if os.path.isfile(os.path.join(d, 'aapt')):
                 aapt = os.path.join(d, 'aapt')
                 break
-        if os.path.isfile(aapt):
+        if aapt and os.path.isfile(aapt):
             dirname = os.path.basename(os.path.dirname(aapt))
             if dirname == 'build-tools':
                 # this is the old layout, before versioned build-tools

hooks/pre-commit

@@ -85,8 +85,10 @@ find_command() {
 	echo :
 }
 
+DASH=$(find_command dash)
 PYFLAKES=$(find_command pyflakes)
 PEP8=$(find_command pycodestyle pep8)
+RUBY=$(find_command ruby)
 
 if [ "$PY_FILES $PY_TEST_FILES" != " " ]; then
     if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then
@@ -110,7 +112,7 @@ if [ "$PY_TEST_FILES" != "" ]; then
 fi
 
 for f in $SH_FILES; do
-	if ! dash -n $f; then
+	if ! $DASH -n $f; then
 		err "dash tests failed!"
 	fi
 done
@@ -122,7 +124,7 @@ for f in $BASH_FILES; do
 done
 
 for f in $RB_FILES; do
-	if ! ruby -c $f 1>/dev/null; then
+	if ! $RUBY -c $f 1>/dev/null; then
 		err "ruby tests failed!"
 	fi
 done

makebuildserver

@@ -553,10 +553,16 @@ def main():
         for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
             fullpath = os.path.join(os.getenv('HOME'), d)
             if os.path.isdir(fullpath):
-                # TODO newer versions of vagrant provide `vagrant rsync`
+                ssh_command = ' '.join('ssh -i {0} -p {1}'.format(key, port),
+                                       '-o StrictHostKeyChecking=no',
+                                       '-o UserKnownHostsFile=/dev/null',
+                                       '-o LogLevel=FATAL',
+                                       '-o IdentitiesOnly=yes',
+                                       '-o PasswordAuthentication=no')
+                # TODO vagrant 1.5+ provides `vagrant rsync`
                 run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
                 subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
-                                 'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
+                                 ssh_command,
                                  fullpath + '/',
                                  user + '@' + hostname + ':~/' + d + '/'])
 

setup.py

@@ -78,6 +78,9 @@ setup(name='fdroidserver',
       data_files=get_data_files(),
       python_requires='>=3.4',
       cmdclass={'versioncheck': VersionCheckCommand},
+      setup_requires=[
+          'babel',
+      ],
       install_requires=[
           'clint',
           'GitPython',

tests/common.TestCase

@@ -86,11 +86,8 @@ class CommonTest(unittest.TestCase):
         sdk_path = os.getenv('ANDROID_HOME')
         if os.path.exists(sdk_path):
             fdroidserver.common.config['sdk_path'] = sdk_path
-            if os.path.exists('/usr/bin/aapt'):
-                # this test only works when /usr/bin/aapt is installed
-                self._find_all()
             build_tools = os.path.join(sdk_path, 'build-tools')
-            if self._set_build_tools():
+            if self._set_build_tools() or os.path.exists('/usr/bin/aapt'):
                 self._find_all()
             else:
                 print('no build-tools found: ' + build_tools)

tests/complete-ci-tests

@@ -74,28 +74,6 @@ for locale in *; do
 done
 
 
-#------------------------------------------------------------------------------#
-# test building the source tarball, then installing it
-cd $WORKSPACE
-python3 setup.py compile_catalog sdist
-
-# make sure translation files got compiled and included
-tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo
-
-rm -rf $WORKSPACE/env
-$pyvenv $WORKSPACE/env
-. $WORKSPACE/env/bin/activate
-# workaround https://github.com/pypa/setuptools/issues/937
-pip3 install --quiet setuptools==33.1.1
-pip3 install --quiet dist/fdroidserver-*.tar.gz
-
-# make sure translation files were installed
-test -e $WORKSPACE/env/share/locale/de/LC_MESSAGES/fdroidserver.mo
-
-# run tests in new pip+pyvenv install
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
-
-
 #------------------------------------------------------------------------------#
 # test install using install direct from git repo
 cd $WORKSPACE