mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-08-17 11:50:11 +02:00
558 lines
18 KiB
YAML
558 lines
18 KiB
YAML
|
|
variables:
|
|
pip: pip3 --timeout 100 --retries 10
|
|
# speed up git checkout phase
|
|
GIT_DEPTH: 1
|
|
|
|
|
|
# Run the whole test suite in an environment that is like the buildserver guest VM.
|
|
ci-images-base run-tests:
|
|
image: registry.gitlab.com/fdroid/ci-images-base
|
|
script:
|
|
- $pip install -e .
|
|
- ./tests/run-tests
|
|
# make sure that translations do not cause stacktraces
|
|
- cd $CI_PROJECT_DIR/locale
|
|
- for locale in *; do
|
|
test -d $locale || continue;
|
|
for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $CI_PROJECT_DIR/fdroid`; do
|
|
LANGUAGE=$locale $CI_PROJECT_DIR/fdroid $cmd --help > /dev/null;
|
|
done
|
|
done
|
|
|
|
# Test that the parsing of the .yml metadata format didn't change from last
|
|
# released version. This uses the commit ID of the release tags,
|
|
# rather than the release tag itself so that contributor forks do not
|
|
# need to include the tags in them for this test to work.
|
|
#
|
|
# The COMMIT_ID should be bumped after each release, so that the list
|
|
# of sed hacks needed does not continuously grow.
|
|
metadata_v0:
|
|
image: registry.gitlab.com/fdroid/ci-images-base
|
|
variables:
|
|
GIT_DEPTH: 1000
|
|
RELEASE_COMMIT_ID: 37c95f59a17d86723fdb71e984121726db777f32 # 2.0a5~
|
|
script:
|
|
- git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
|
|
- cd tests
|
|
- export GITCOMMIT=`git describe`
|
|
- git checkout $RELEASE_COMMIT_ID
|
|
- cd ..
|
|
- git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
|
|
- cd fdroiddata
|
|
- ../tests/dump_internal_metadata_format.py
|
|
- cd ..
|
|
- git reset --hard
|
|
- git checkout $GITCOMMIT
|
|
- cd fdroiddata
|
|
- ../tests/dump_internal_metadata_format.py
|
|
- sed -i
|
|
-e '/AllowedAPKSigningKeys:/d'
|
|
-e '/Liberapay:/d'
|
|
-e '/OpenCollective/d'
|
|
metadata/dump_*/*.yaml
|
|
- diff -uw metadata/dump_*
|
|
|
|
.apt-template: &apt-template
|
|
variables:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
LANG: C.UTF-8
|
|
before_script:
|
|
- echo Etc/UTC > /etc/timezone
|
|
- echo 'APT::Install-Recommends "0";'
|
|
'APT::Install-Suggests "0";'
|
|
'APT::Acquire::Retries "20";'
|
|
'APT::Get::Assume-Yes "true";'
|
|
'Dpkg::Use-Pty "0";'
|
|
'quiet "1";'
|
|
>> /etc/apt/apt.conf.d/99gitlab
|
|
- apt-get update
|
|
- apt-get dist-upgrade
|
|
|
|
|
|
# Since F-Droid uses Debian as its default platform, from production
|
|
# servers to CI to contributor machines, it is important to know when
|
|
# changes in Debian break our stuff. This tests against the latest
|
|
# dependencies as they are included in Debian.
|
|
debian_testing:
|
|
image: debian:testing
|
|
<<: *apt-template
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- apt-get install
|
|
aapt
|
|
androguard
|
|
apksigner
|
|
fdroidserver
|
|
git
|
|
gnupg
|
|
python3-defusedxml
|
|
python3-setuptools
|
|
zipalign
|
|
- python3 -c 'import fdroidserver'
|
|
- python3 -c 'import androguard'
|
|
- cd tests
|
|
- ./run-tests
|
|
|
|
|
|
# Test using latest LTS set up with the PPA, including Recommends.
|
|
ubuntu_lts_ppa:
|
|
image: ubuntu:latest
|
|
<<: *apt-template
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- export ANDROID_HOME=/usr/lib/android-sdk
|
|
- apt-get install gnupg
|
|
- while ! apt-key adv --keyserver keyserver.ubuntu.com --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done
|
|
- export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
|
|
- echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
|
|
- apt-get update
|
|
- apt-get dist-upgrade
|
|
- apt-get install --install-recommends fdroidserver git python3-setuptools
|
|
- cd tests
|
|
- ./run-tests
|
|
|
|
|
|
# Test using Ubuntu/bionic LTS (supported til 2022) with all depends
|
|
# from pypi. The venv is used to isolate the dist tarball generation
|
|
# environment from the clean install environment.
|
|
ubuntu_bionic_pip:
|
|
image: ubuntu:bionic
|
|
<<: *apt-template
|
|
script:
|
|
- apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13
|
|
- rm -rf env
|
|
- pyvenv env
|
|
- . env/bin/activate
|
|
- $pip install --upgrade babel pip setuptools
|
|
# setup venv to act as release build machine
|
|
- python -m venv sdist-env
|
|
- . sdist-env/bin/activate
|
|
- ./setup.py sdist
|
|
- deactivate
|
|
- tar tzf dist/fdroidserver-*.tar.gz
|
|
# back to bare machine to act as user's install machine
|
|
- $pip install --upgrade pip setuptools wheel # make this go away: "error: invalid command 'bdist_wheel'"
|
|
- $pip install dist/fdroidserver-*.tar.gz
|
|
- tar xzf dist/fdroidserver-*.tar.gz
|
|
- cd fdroidserver-*
|
|
- fdroid=`which fdroid` ./tests/run-tests
|
|
|
|
|
|
# test installation process on a bleeding edge distro with pip
|
|
arch_pip_install:
|
|
image: archlinux
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- pacman --sync --sysupgrade --refresh --noconfirm gcc git grep python-pip python-virtualenv python-wheel tar
|
|
- pip install -e .[test]
|
|
- fdroid
|
|
- fdroid readmeta
|
|
- fdroid update --help
|
|
|
|
|
|
# The gradlew-fdroid tests are isolated from the rest of the test
|
|
# suite, so they run as their own job.
|
|
gradlew-fdroid:
|
|
image: debian:bullseye
|
|
<<: *apt-template
|
|
only:
|
|
changes:
|
|
- .gitlab-ci.yml
|
|
- gradlew-fdroid
|
|
- tests/test-gradlew-fdroid
|
|
script:
|
|
- apt-get install ca-certificates curl default-jdk-headless unzip
|
|
- ./tests/test-gradlew-fdroid
|
|
|
|
|
|
# Run all the various linters and static analysis tools.
|
|
lint_format_safety_bandit_checks:
|
|
image: alpine:3.13
|
|
variables:
|
|
LANG: C.UTF-8
|
|
script:
|
|
- apk add --no-cache bash build-base dash ca-certificates gcc python3 python3-dev
|
|
- python3 -m ensurepip
|
|
- $pip install Babel bandit pycodestyle pyflakes pylint safety
|
|
- export EXITVALUE=0
|
|
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
|
|
- ./hooks/pre-commit || set_error
|
|
- bandit
|
|
-r
|
|
-ii
|
|
--ini .bandit
|
|
|| set_error
|
|
- safety check --full-report || set_error
|
|
- pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n
|
|
fdroid
|
|
makebuildserver
|
|
setup.py
|
|
fdroidserver/*.py
|
|
tests/*.py
|
|
tests/*.TestCase
|
|
|| set_error
|
|
- apk add --no-cache gettext make
|
|
- make -C locale compile || set_error
|
|
- rm -f locale/*/*/*.mo
|
|
- pybabel compile --domain=fdroidserver --directory locale 2>&1 | (grep -F "error:" && exit 1) || true
|
|
- exit $EXITVALUE
|
|
|
|
|
|
lint_mypy:
|
|
image: debian:bullseye-backports
|
|
<<: *apt-template
|
|
script:
|
|
# use Debian packages to avoid building C/rust sources
|
|
- apt-get install
|
|
mypy
|
|
python3-cryptography
|
|
python3-pip
|
|
python3-wheel
|
|
- apt-get install -t bullseye-backports python3-pyjks
|
|
- pip install -e .[test]
|
|
- mypy
|
|
|
|
|
|
black:
|
|
image: python:slim
|
|
script:
|
|
- pip install black
|
|
- black --check --diff --color --skip-string-normalization
|
|
examples/fdroid_extract_repo_pubkey.py
|
|
examples/makebuildserver.config.py
|
|
fdroid
|
|
fdroidserver/exception.py
|
|
fdroidserver/gpgsign.py
|
|
fdroidserver/lint.py
|
|
fdroidserver/mirror.py
|
|
fdroidserver/net.py
|
|
fdroidserver/readmeta.py
|
|
fdroidserver/signindex.py
|
|
fdroidserver/tail.py
|
|
setup.py
|
|
tests/build.TestCase
|
|
tests/deploy.TestCase
|
|
tests/exception.TestCase
|
|
tests/import.TestCase
|
|
tests/import_proxy.py
|
|
tests/init.TestCase
|
|
tests/install.TestCase
|
|
tests/key-tricks.py
|
|
tests/lint.TestCase
|
|
tests/metadata.TestCase
|
|
tests/ndk-release-checksums.py
|
|
tests/rewritemeta.TestCase
|
|
|
|
|
|
fedora_latest:
|
|
image: fedora:latest
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
# tricks to hopefully make runs more reliable
|
|
- echo "timeout=600" >> /etc/dnf/dnf.conf
|
|
- echo "retries=50" >> /etc/dnf/dnf.conf
|
|
- echo "keepcache=True" >> /etc/dnf/dnf.conf
|
|
|
|
- dnf -y update || dnf -y update
|
|
- dnf -y install @development-tools
|
|
diffutils
|
|
findutils
|
|
git
|
|
gnupg
|
|
java-1.8.0-openjdk-devel
|
|
python3
|
|
python3-babel
|
|
python3-matplotlib
|
|
python3-pip
|
|
rsync
|
|
unzip
|
|
wget
|
|
which
|
|
- ./setup.py sdist
|
|
- useradd -m -c "test account" --password "fakepassword" testuser
|
|
- su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
|
|
- test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
|
|
- wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip
|
|
- unzip -q tools.zip
|
|
- rm tools.zip
|
|
- export BUILD_TOOLS_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
|
|
- export JAVA_HOME=/etc/alternatives/jre
|
|
- export ANDROID_HOME=`pwd`/android-sdk
|
|
- mkdir $ANDROID_HOME
|
|
- mv tools $ANDROID_HOME/
|
|
- mkdir -p $ANDROID_HOME/licenses/
|
|
- printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e\n24333f8a63b6825ea9c5514f83c2829b004d1fee" > $ANDROID_HOME/licenses/android-sdk-license
|
|
- printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
|
|
- printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old
|
|
- mkdir ~/.android
|
|
- touch ~/.android/repositories.cfg
|
|
- echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools" > /dev/null
|
|
- echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION" > /dev/null
|
|
- chown -R testuser .
|
|
- cd tests
|
|
- su testuser --login --command
|
|
"cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
|
|
|
|
|
|
gradle/ndk:
|
|
image: debian:bullseye
|
|
<<: *apt-template
|
|
variables:
|
|
GIT_DEPTH: 1000
|
|
script:
|
|
- apt-get install
|
|
ca-certificates
|
|
git
|
|
openssh-client
|
|
python3-bs4
|
|
python3-colorama
|
|
python3-git
|
|
python3-gitlab
|
|
python3-requests
|
|
# if this is a merge request fork, then only check if relevant files changed
|
|
- if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
|
|
git fetch https://gitlab.com/fdroid/fdroidserver.git;
|
|
for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
|
|
test "$f" == "makebuildserver" && export CHANGED="yes";
|
|
test "$f" == "gradlew-fdroid" && export CHANGED="yes";
|
|
test "$f" == "fdroidserver/common.py" && export CHANGED="yes";
|
|
done;
|
|
test -z "$CHANGED" && exit;
|
|
fi
|
|
- ./tests/gradle-release-checksums.py
|
|
- ./tests/ndk-release-checksums.py
|
|
|
|
|
|
# Run an actual build in a simple, faked version of the buildserver guest VM.
|
|
fdroid build:
|
|
image: registry.gitlab.com/fdroid/ci-images-client
|
|
only:
|
|
changes:
|
|
- .gitlab-ci.yml
|
|
- buildserver/provision-apt-get-install
|
|
- fdroidserver/build.py
|
|
- fdroidserver/common.py
|
|
- fdroidserver/exception.py
|
|
- fdroidserver/metadata.py
|
|
- fdroidserver/net.py
|
|
- fdroidserver/scanner.py
|
|
- fdroidserver/vmtools.py
|
|
cache:
|
|
key: "$CI_JOB_NAME"
|
|
paths:
|
|
- .gradle
|
|
script:
|
|
- bash buildserver/provision-apt-get-install http://deb.debian.org/debian
|
|
- apt-get dist-upgrade
|
|
- apt-get install -t stretch-backports
|
|
python3-asn1crypto
|
|
python3-pip
|
|
python3-ruamel.yaml
|
|
python3-setuptools
|
|
python3-venv
|
|
- apt-get purge fdroidserver
|
|
- pyvenv env --system-site-packages
|
|
- . env/bin/activate
|
|
- $pip install -e .
|
|
- export PATH="$CI_PROJECT_DIR:$PATH"
|
|
- export PYTHONPATH=$CI_PROJECT_DIR
|
|
- export PYTHONUNBUFFERED=true
|
|
|
|
- git clone https://gitlab.com/fdroid/fdroiddata.git --depth 1
|
|
- cd fdroiddata
|
|
- test -d build || mkdir build
|
|
|
|
- export GRADLE_USER_HOME=$CI_PROJECT_DIR/.gradle
|
|
# try a user build first
|
|
- fdroid build --verbose --latest org.fdroid.fdroid.privileged
|
|
# each `fdroid build --on-server` run expects sudo, then uninstalls it
|
|
- apt-get install sudo
|
|
- currentVersionCode=`python3 -c "import yaml; print(yaml.load(open('metadata/org.fdroid.fdroid.yml'))['CurrentVersionCode'])"`
|
|
- fdroid build --verbose --on-server --no-tarball org.fdroid.fdroid:$currentVersionCode
|
|
|
|
|
|
# test the plugin API and specifically the fetchsrclibs plugin, which
|
|
# is used by the `fdroid build` job. This uses a fixed commit from
|
|
# fdroiddata because that one is known to work, and this is a CI job,
|
|
# so it should be isolated from the normal churn of fdroiddata.
|
|
plugin_fetchsrclibs:
|
|
image: debian:buster
|
|
<<: *apt-template
|
|
only:
|
|
changes:
|
|
- .gitlab-ci.yml
|
|
- examples/fdroid_fetchsrclibs.py
|
|
- fdroidserver/__main__.py
|
|
script:
|
|
- apt-get install
|
|
curl
|
|
git
|
|
python3-cffi
|
|
python3-cryptography
|
|
python3-matplotlib
|
|
python3-nacl
|
|
python3-pip
|
|
python3-pil
|
|
python3-pycparser
|
|
python3-venv
|
|
- python3 -m venv --system-site-packages env
|
|
- . env/bin/activate
|
|
- export PATH="$CI_PROJECT_DIR:$PATH"
|
|
- export PYTHONPATH="$CI_PROJECT_DIR/examples"
|
|
- $pip install wheel # to make this go away: "error: invalid command 'bdist_wheel'"
|
|
- $pip install -e .
|
|
- fdroid | grep fetchsrclibs
|
|
|
|
- mkdir fdroiddata
|
|
- commitid=b9e9a077d720c86ff6fff4dbb341254cc4370b1a
|
|
- curl https://gitlab.com/fdroid/fdroiddata/-/archive/${commitid}/fdroiddata-${commitid}.tar.gz
|
|
| tar -xz --directory=fdroiddata --strip-components=1
|
|
- cd fdroiddata
|
|
- fdroid fetchsrclibs freemap.opentrail:9 --verbose
|
|
- test -d build/freemap.opentrail/.git
|
|
- test -d build/srclib/andromaps/.git
|
|
- test -d build/srclib/freemaplib/.git
|
|
- test -d build/srclib/freemaplibProj/.git
|
|
- test -d build/srclib/JCoord/.git
|
|
- test -d build/srclib/javaproj/.git
|
|
|
|
|
|
# test a full update and deploy cycle to gitlab.com
|
|
servergitmirrors:
|
|
image: debian:buster
|
|
<<: *apt-template
|
|
only:
|
|
- master@fdroid/fdroidserver
|
|
script:
|
|
- apt-get install
|
|
default-jdk-headless
|
|
git
|
|
openssh-client
|
|
openssl
|
|
python3-cffi
|
|
python3-cryptography
|
|
python3-matplotlib
|
|
python3-nacl
|
|
python3-pil
|
|
python3-pip
|
|
python3-pycparser
|
|
python3-venv
|
|
rsync
|
|
wget
|
|
- python3 -m venv --system-site-packages env
|
|
- . env/bin/activate
|
|
- export PYTHONPATH=`pwd`
|
|
- $pip install -e .
|
|
- mkdir /root/.ssh/
|
|
- ./tests/key-tricks.py
|
|
- ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
|
|
- test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
|
|
- cp tests/config.py tests/keystore.jks /tmp/fdroid/
|
|
- cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
|
|
- cd /tmp/fdroid
|
|
- touch fdroid-icon.png
|
|
- printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
|
|
- $PYTHONPATH/fdroid update --verbose --create-metadata
|
|
- $PYTHONPATH/fdroid deploy --verbose
|
|
- export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
|
|
- echo $DLURL
|
|
- wget $DLURL/index-v1.jar
|
|
- diff repo/index-v1.jar index-v1.jar
|
|
|
|
Build documentation:
|
|
image: python:3.9-buster
|
|
script:
|
|
- pip install -e .[docs]
|
|
- pydocstyle fdroidserver
|
|
- cd docs
|
|
- sphinx-apidoc -o ./source ../fdroidserver -M -e
|
|
- sphinx-autogen -o generated source/*.rst
|
|
- make html
|
|
artifacts:
|
|
paths:
|
|
- docs/build/html/
|
|
|
|
|
|
Windows:
|
|
tags:
|
|
- windows
|
|
script:
|
|
- Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
|
|
- choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath"
|
|
- choco install --no-progress -y python3
|
|
- choco install --no-progress -y jdk8
|
|
- choco install --no-progress -y rsync
|
|
- refreshenv
|
|
- python -m pip install --upgrade babel pip setuptools
|
|
- python -m pip install -e .
|
|
|
|
- $files = @(Get-ChildItem tests\*.TestCase)
|
|
- foreach ($f in $files) {
|
|
write-output $f;
|
|
python $f;
|
|
if( $LASTEXITCODE -eq 0 ) {
|
|
write-output "SUCCESS $f";
|
|
} else {
|
|
write-output "ERROR $f failed";
|
|
}
|
|
}
|
|
|
|
# these are the tests that must pass
|
|
- python tests\checkupdates.TestCase
|
|
- python tests\exception.TestCase
|
|
- python tests\import.TestCase
|
|
- python tests\init.TestCase
|
|
- python tests\lint.TestCase
|
|
- python tests\main.TestCase
|
|
- python tests\metadata.TestCase
|
|
- python tests\rewritemeta.TestCase
|
|
- python tests\vcs.TestCase
|
|
after_script:
|
|
- Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log
|
|
artifacts:
|
|
when: always
|
|
paths:
|
|
- "*.log"
|
|
allow_failure:
|
|
exit_codes: 1
|
|
|
|
|
|
pages:
|
|
image: alpine:latest
|
|
stage: deploy
|
|
script:
|
|
- cp docs/build/html public -r # GL Pages needs the files in a directory named "public"
|
|
artifacts:
|
|
paths:
|
|
- public
|
|
needs: ["Build documentation"]
|
|
rules:
|
|
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # only publish pages on default (master) branch
|
|
|
|
|
|
docker:
|
|
dependencies:
|
|
- fdroid build
|
|
only:
|
|
changes:
|
|
- .gitlab-ci.yml
|
|
- makebuildserver
|
|
- buildserver/*
|
|
image: docker:git
|
|
services:
|
|
- docker:dind
|
|
variables:
|
|
TEST_IMAGE: registry.gitlab.com/$CI_PROJECT_NAMESPACE/${CI_PROJECT_NAME}:$CI_BUILD_REF_NAME
|
|
RELEASE_IMAGE: registry.gitlab.com/$CI_PROJECT_NAMESPACE/${CI_PROJECT_NAME}:buildserver
|
|
script:
|
|
- cd buildserver
|
|
- docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) .
|
|
- docker tag $TEST_IMAGE $RELEASE_IMAGE
|
|
- echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com
|
|
- docker push $RELEASE_IMAGE
|