mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-08-17 11:50:11 +02:00
mypy 0.901 was released and everything broke: https://gitlab.com/linsui/fdroidserver/-/jobs/1330206567 My point is to reduce the number of false job fails like this one. We have a lot of checkers running, I really think they need to prove they are adding value before we invest any time maintaining them. mypy is still "wait and see" in terms of the adding any value. !951
449 lines
15 KiB
YAML
449 lines
15 KiB
YAML
|
|
variables:
|
|
pip: pip3 --timeout 100 --retries 10
|
|
# speed up git checkout phase
|
|
GIT_DEPTH: 1
|
|
|
|
|
|
# Run the whole test suite in an environment that is like the buildserver guest VM.
|
|
ci-images-base run-tests:
|
|
image: registry.gitlab.com/fdroid/ci-images-base
|
|
script:
|
|
- $pip install -e .
|
|
- ./tests/run-tests
|
|
# make sure that translations do not cause stacktraces
|
|
- cd $CI_PROJECT_DIR/locale
|
|
- for locale in *; do
|
|
test -d $locale || continue;
|
|
for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $CI_PROJECT_DIR/fdroid`; do
|
|
LANGUAGE=$locale $CI_PROJECT_DIR/fdroid $cmd --help > /dev/null;
|
|
done
|
|
done
|
|
|
|
# Test that the parsing of the .yml metadata format didn't change from last
|
|
# released version. This uses the commit ID of the release tags,
|
|
# rather than the release tag itself so that contributor forks do not
|
|
# need to include the tags in them for this test to work.
|
|
#
|
|
# The COMMIT_ID should be bumped after each release, so that the list
|
|
# of sed hacks needed does not continuously grow.
|
|
metadata_v0:
|
|
image: registry.gitlab.com/fdroid/ci-images-base
|
|
variables:
|
|
GIT_DEPTH: 1000
|
|
RELEASE_COMMIT_ID: 37c95f59a17d86723fdb71e984121726db777f32 # 2.0a5~
|
|
script:
|
|
- git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
|
|
- cd tests
|
|
- export GITCOMMIT=`git describe`
|
|
- git checkout $RELEASE_COMMIT_ID
|
|
- cd ..
|
|
- git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
|
|
- cd fdroiddata
|
|
- ../tests/dump_internal_metadata_format.py
|
|
- cd ..
|
|
- git reset --hard
|
|
- git checkout $GITCOMMIT
|
|
- cd fdroiddata
|
|
- ../tests/dump_internal_metadata_format.py
|
|
- sed -i
|
|
-e '/Liberapay:/d'
|
|
-e '/OpenCollective/d'
|
|
metadata/dump_*/*.yaml
|
|
- diff -uw metadata/dump_*
|
|
|
|
.apt-template: &apt-template
|
|
variables:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
LANG: C.UTF-8
|
|
before_script:
|
|
- echo Etc/UTC > /etc/timezone
|
|
- echo 'APT::Install-Recommends "0";'
|
|
'APT::Install-Suggests "0";'
|
|
'APT::Acquire::Retries "20";'
|
|
'APT::Get::Assume-Yes "true";'
|
|
'Dpkg::Use-Pty "0";'
|
|
'quiet "1";'
|
|
>> /etc/apt/apt.conf.d/99gitlab
|
|
- apt-get update
|
|
- apt-get dist-upgrade
|
|
|
|
|
|
# Since F-Droid uses Debian as its default platform, from production
|
|
# servers to CI to contributor machines, it is important to know when
|
|
# changes in Debian break our stuff. This tests against the latest
|
|
# dependencies as they are included in Debian.
|
|
debian_testing:
|
|
image: debian:testing
|
|
<<: *apt-template
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- apt-get install
|
|
aapt
|
|
androguard
|
|
apksigner
|
|
fdroidserver
|
|
git
|
|
gnupg
|
|
python3-defusedxml
|
|
python3-setuptools
|
|
zipalign
|
|
- python3 -c 'import fdroidserver'
|
|
- python3 -c 'import androguard'
|
|
- cd tests
|
|
- ./run-tests
|
|
|
|
|
|
# Test using latest LTS set up with the PPA, including Recommends.
|
|
ubuntu_lts_ppa:
|
|
image: ubuntu:latest
|
|
<<: *apt-template
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- export ANDROID_HOME=/usr/lib/android-sdk
|
|
- apt-get install gnupg
|
|
- while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done
|
|
- export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
|
|
- echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
|
|
- apt-get update
|
|
- apt-get dist-upgrade
|
|
- apt-get install --install-recommends fdroidserver git python3-setuptools
|
|
- cd tests
|
|
- ./run-tests
|
|
|
|
|
|
# Test using Ubuntu/bionic LTS (supported til 2022) with all depends
|
|
# from pypi. The venv is used to isolate the dist tarball generation
|
|
# environment from the clean install environment.
|
|
ubuntu_bionic_pip:
|
|
image: ubuntu:bionic
|
|
<<: *apt-template
|
|
script:
|
|
- apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13
|
|
- rm -rf env
|
|
- pyvenv env
|
|
- . env/bin/activate
|
|
- $pip install --upgrade babel pip setuptools
|
|
# setup venv to act as release build machine
|
|
- python -m venv sdist-env
|
|
- . sdist-env/bin/activate
|
|
- ./setup.py compile_catalog sdist
|
|
- deactivate
|
|
- tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo
|
|
# back to bare machine to act as user's install machine
|
|
- $pip install --upgrade pip setuptools wheel # make this go away: "error: invalid command 'bdist_wheel'"
|
|
- $pip install dist/fdroidserver-*.tar.gz
|
|
- tar xzf dist/fdroidserver-*.tar.gz
|
|
- cd fdroidserver-*
|
|
- fdroid=`which fdroid` ./tests/run-tests
|
|
|
|
|
|
# test install process on a bleeding edge distro with pip
|
|
arch_pip_install:
|
|
image: archlinux
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- pacman --sync --sysupgrade --refresh --noconfirm gcc git grep python-pip python-virtualenv python-wheel tar
|
|
- pip install -e .[test]
|
|
- fdroid
|
|
- fdroid readmeta
|
|
- fdroid update --help
|
|
|
|
|
|
# The gradlew-fdroid tests are isolated from the rest of the test
|
|
# suite, so they run as their own job.
|
|
gradlew-fdroid:
|
|
image: debian:bullseye
|
|
<<: *apt-template
|
|
only:
|
|
changes:
|
|
- .gitlab-ci.yml
|
|
- gradlew-fdroid
|
|
- tests/test-gradlew-fdroid
|
|
script:
|
|
- apt-get install ca-certificates curl default-jdk-headless unzip
|
|
- ./tests/test-gradlew-fdroid
|
|
|
|
|
|
# Run all the various linters and static analysis tools.
|
|
lint_format_safety_bandit_checks:
|
|
image: alpine:3.13
|
|
variables:
|
|
LANG: C.UTF-8
|
|
script:
|
|
- apk add --no-cache bash build-base dash ca-certificates gcc python3 python3-dev
|
|
- python3 -m ensurepip
|
|
- $pip install Babel bandit pycodestyle pyflakes pylint safety
|
|
- export EXITVALUE=0
|
|
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
|
|
- ./hooks/pre-commit || set_error
|
|
- bandit
|
|
-ii
|
|
-s B110,B404,B408,B410,B603,B607
|
|
-r $CI_PROJECT_DIR fdroid
|
|
|| set_error
|
|
- safety check --full-report || set_error
|
|
- pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n
|
|
fdroid
|
|
makebuildserver
|
|
setup.py
|
|
fdroidserver/*.py
|
|
tests/*.py
|
|
tests/*.TestCase
|
|
|| set_error
|
|
- apk add --no-cache gettext make
|
|
- make -C locale compile || set_error
|
|
- rm -f locale/*/*/*.mo
|
|
- pybabel compile --domain=fdroidserver --directory locale 2>&1 | (grep -F "error:" && exit 1) || true
|
|
- exit $EXITVALUE
|
|
|
|
|
|
lint_mypy:
|
|
image: debian:bullseye
|
|
<<: *apt-template
|
|
script:
|
|
- sed -i '/pyjks/d' setup.py # TODO get from backports once available
|
|
# use Debian packages to avoid building C/rust sources
|
|
- apt-get install
|
|
mypy
|
|
python3-cryptography
|
|
python3-pip
|
|
python3-wheel
|
|
- pip install -e .[test]
|
|
# exclude vendored file
|
|
- mypy --exclude fdroidserver/apksigcopier.py
|
|
|
|
|
|
fedora_latest:
|
|
image: fedora:latest
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
# tricks to hopefully make runs more reliable
|
|
- echo "timeout=600" >> /etc/dnf/dnf.conf
|
|
- echo "retries=50" >> /etc/dnf/dnf.conf
|
|
- echo "keepcache=True" >> /etc/dnf/dnf.conf
|
|
|
|
- dnf -y update || dnf -y update
|
|
- dnf -y install @development-tools
|
|
diffutils
|
|
findutils
|
|
git
|
|
gnupg
|
|
java-1.8.0-openjdk-devel
|
|
python3
|
|
python3-babel
|
|
python3-matplotlib
|
|
python3-pip
|
|
rsync
|
|
unzip
|
|
wget
|
|
which
|
|
- ./setup.py compile_catalog sdist
|
|
- useradd -m -c "test account" --password "fakepassword" testuser
|
|
- su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
|
|
- test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
|
|
- wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip
|
|
- unzip -q tools.zip
|
|
- rm tools.zip
|
|
- export BUILD_TOOLS_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
|
|
- export JAVA_HOME=/etc/alternatives/jre
|
|
- export ANDROID_HOME=`pwd`/android-sdk
|
|
- mkdir $ANDROID_HOME
|
|
- mv tools $ANDROID_HOME/
|
|
- mkdir -p $ANDROID_HOME/licenses/
|
|
- printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e\n24333f8a63b6825ea9c5514f83c2829b004d1fee" > $ANDROID_HOME/licenses/android-sdk-license
|
|
- printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
|
|
- printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old
|
|
- mkdir ~/.android
|
|
- touch ~/.android/repositories.cfg
|
|
- echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools" > /dev/null
|
|
- echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION" > /dev/null
|
|
- chown -R testuser .
|
|
- cd tests
|
|
- su testuser --login --command
|
|
"cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
|
|
|
|
|
|
gradle/ndk:
|
|
image: debian:bullseye
|
|
<<: *apt-template
|
|
variables:
|
|
GIT_DEPTH: 1000
|
|
script:
|
|
- apt-get install
|
|
ca-certificates
|
|
git
|
|
openssh-client
|
|
python3-bs4
|
|
python3-colorama
|
|
python3-git
|
|
python3-gitlab
|
|
python3-requests
|
|
# if this is a merge request fork, then only check if relevant files changed
|
|
- if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
|
|
git fetch https://gitlab.com/fdroid/fdroidserver.git;
|
|
for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
|
|
test "$f" == "makebuildserver" && export CHANGED="yes";
|
|
test "$f" == "gradlew-fdroid" && export CHANGED="yes";
|
|
test "$f" == "fdroidserver/common.py" && export CHANGED="yes";
|
|
done;
|
|
test -z "$CHANGED" && exit;
|
|
fi
|
|
- ./tests/gradle-release-checksums.py
|
|
- ./tests/ndk-release-checksums.py
|
|
|
|
|
|
# Run an actual build in a simple, faked version of the buildserver guest VM.
|
|
fdroid build:
|
|
image: registry.gitlab.com/fdroid/ci-images-client
|
|
only:
|
|
changes:
|
|
- .gitlab-ci.yml
|
|
- buildserver/provision-apt-get-install
|
|
- fdroidserver/build.py
|
|
- fdroidserver/common.py
|
|
- fdroidserver/exception.py
|
|
- fdroidserver/metadata.py
|
|
- fdroidserver/net.py
|
|
- fdroidserver/scanner.py
|
|
- fdroidserver/vmtools.py
|
|
cache:
|
|
key: "$CI_JOB_NAME"
|
|
paths:
|
|
- .gradle
|
|
script:
|
|
- bash buildserver/provision-apt-get-install http://deb.debian.org/debian
|
|
- apt-get dist-upgrade
|
|
- apt-get install -t stretch-backports
|
|
python3-asn1crypto
|
|
python3-pip
|
|
python3-ruamel.yaml
|
|
python3-setuptools
|
|
python3-venv
|
|
- apt-get purge fdroidserver
|
|
- pyvenv env --system-site-packages
|
|
- . env/bin/activate
|
|
- $pip install -e .
|
|
- export PATH="$CI_PROJECT_DIR:$PATH"
|
|
- export PYTHONPATH=$CI_PROJECT_DIR
|
|
- export PYTHONUNBUFFERED=true
|
|
|
|
- git clone https://gitlab.com/fdroid/fdroiddata.git --depth 1
|
|
- cd fdroiddata
|
|
- test -d build || mkdir build
|
|
|
|
- export GRADLE_USER_HOME=$CI_PROJECT_DIR/.gradle
|
|
# try a user build first
|
|
- fdroid build --verbose --latest org.fdroid.fdroid.privileged
|
|
# each `fdroid build --on-server` run expects sudo, then uninstalls it
|
|
- apt-get install sudo
|
|
- fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid
|
|
|
|
|
|
# test the plugin API and specifically the fetchsrclibs plugin, which
|
|
# is used by the `fdroid build` job. This uses a fixed commit from
|
|
# fdroiddata because that one is known to work, and this is a CI job,
|
|
# so it should be isolated from the normal churn of fdroiddata.
|
|
plugin_fetchsrclibs:
|
|
image: debian:buster
|
|
<<: *apt-template
|
|
only:
|
|
changes:
|
|
- .gitlab-ci.yml
|
|
- examples/fdroid_fetchsrclibs.py
|
|
- fdroidserver/__main__.py
|
|
script:
|
|
- apt-get install
|
|
curl
|
|
git
|
|
python3-cryptography
|
|
python3-pip
|
|
python3-venv
|
|
- python3 -m venv --system-site-packages env
|
|
- . env/bin/activate
|
|
- export PATH="$CI_PROJECT_DIR:$PATH"
|
|
- export PYTHONPATH="$CI_PROJECT_DIR/examples"
|
|
- $pip install wheel # to make this go away: "error: invalid command 'bdist_wheel'"
|
|
- $pip install -e .
|
|
- fdroid | grep fetchsrclibs
|
|
|
|
- mkdir fdroiddata
|
|
- commitid=b9e9a077d720c86ff6fff4dbb341254cc4370b1a
|
|
- curl https://gitlab.com/fdroid/fdroiddata/-/archive/${commitid}/fdroiddata-${commitid}.tar.gz
|
|
| tar -xz --directory=fdroiddata --strip-components=1
|
|
- cd fdroiddata
|
|
- fdroid fetchsrclibs freemap.opentrail:9 --verbose
|
|
- test -d build/freemap.opentrail/.git
|
|
- test -d build/srclib/andromaps/.git
|
|
- test -d build/srclib/freemaplib/.git
|
|
- test -d build/srclib/freemaplibProj/.git
|
|
- test -d build/srclib/JCoord/.git
|
|
- test -d build/srclib/javaproj/.git
|
|
|
|
|
|
# test a full update and deploy cycle to gitlab.com
|
|
servergitmirrors:
|
|
image: debian:buster
|
|
<<: *apt-template
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- apt-get install
|
|
default-jdk-headless
|
|
git
|
|
openssh-client
|
|
openssl
|
|
python3-cryptography
|
|
python3-pip
|
|
python3-venv
|
|
rsync
|
|
wget
|
|
- python3 -m venv --system-site-packages env
|
|
- . env/bin/activate
|
|
- export PYTHONPATH=`pwd`
|
|
- $pip install -e .
|
|
- mkdir /root/.ssh/
|
|
- ./tests/key-tricks.py
|
|
- ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
|
|
- test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
|
|
- cp tests/config.py tests/keystore.jks /tmp/fdroid/
|
|
- cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
|
|
- cd /tmp/fdroid
|
|
- touch fdroid-icon.png
|
|
- printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
|
|
- $PYTHONPATH/fdroid update --verbose --create-metadata
|
|
- $PYTHONPATH/fdroid deploy --verbose
|
|
- export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
|
|
- echo $DLURL
|
|
- wget $DLURL/index-v1.jar
|
|
- diff repo/index-v1.jar index-v1.jar
|
|
|
|
Build documentation:
|
|
image: python:3.9-buster
|
|
script:
|
|
- pip install -e .[docs]
|
|
- cd docs
|
|
- sphinx-apidoc -o ./source ../fdroidserver -M -e
|
|
- sphinx-autogen -o generated source/*.rst
|
|
- make html
|
|
artifacts:
|
|
paths:
|
|
- docs/build/html/
|
|
|
|
|
|
pages:
|
|
image: alpine:latest
|
|
stage: deploy
|
|
script:
|
|
- cp docs/build/html public -r # GL Pages needs the files in a directory named "public"
|
|
artifacts:
|
|
paths:
|
|
- public
|
|
needs: ["Build documentation"]
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # only publish pages on default (master) branch
|