diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50927c6a..88464987 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -395,8 +395,10 @@ plugin_fetchsrclibs: python3-cffi python3-cryptography python3-matplotlib + python3-nacl python3-pip python3-pil + python3-pycparser python3-venv - python3 -m venv --system-site-packages env - . env/bin/activate @@ -529,3 +531,25 @@ pages: needs: ["Build documentation"] rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # only publish pages on default (master) branch + + +docker: + dependencies: + - fdroid build + only: + changes: + - .gitlab-ci.yml + - makebuildserver + - buildserver/* + image: docker:git + services: + - docker:dind + variables: + TEST_IMAGE: registry.gitlab.com/$CI_PROJECT_NAMESPACE/${CI_PROJECT_NAME}:$CI_BUILD_REF_NAME + RELEASE_IMAGE: registry.gitlab.com/$CI_PROJECT_NAMESPACE/${CI_PROJECT_NAME}:buildserver + script: + - cd buildserver + - docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) . + - docker tag $TEST_IMAGE $RELEASE_IMAGE + - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com + - docker push $RELEASE_IMAGE diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile new file mode 100644 index 00000000..0d968a07 --- /dev/null +++ b/buildserver/Dockerfile @@ -0,0 +1,59 @@ + +FROM debian:stretch + +ENV LANG=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive + +RUN echo Etc/UTC > /etc/timezone \ + && echo 'APT::Install-Recommends "0";' \ + 'APT::Install-Suggests "0";' \ + 'APT::Acquire::Retries "20";' \ + 'APT::Get::Assume-Yes "true";' \ + 'Dpkg::Use-Pty "0";' \ + 'quiet "1";' \ + >> /etc/apt/apt.conf.d/99gitlab + +# provision-apt-proxy was deliberately omitted, its not relevant in Docker +COPY provision-android-ndk \ + provision-android-sdk \ + provision-apt-get-install \ + provision-buildserverid \ + provision-gradle \ + setup-env-vars \ + /opt/buildserver/ + +ARG GIT_REV_PARSE_HEAD=unspecified +LABEL org.opencontainers.image.revision=$GIT_REV_PARSE_HEAD + +# setup 'vagrant' user for compatibility +RUN useradd --create-home -s /bin/bash vagrant && echo -n 'vagrant:vagrant' | chpasswd + +# the provision scripts must be run in the same order as in Vagrantfile +# - vagrant needs openssh-client iproute2 ssh sudo +# - ansible needs python3 +RUN printf "path-exclude=/usr/share/locale/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-include=/usr/share/doc/*/copyright\n" >/etc/dpkg/dpkg.cfg.d/01_nodoc \ + && mkdir -p /usr/share/man/man1 \ + && apt-get update \ + && apt-get upgrade \ + && apt-get dist-upgrade \ + && apt-get install openssh-client iproute2 python3 openssh-server sudo \ + && bash /opt/buildserver/setup-env-vars /opt/android-sdk \ + && . /etc/profile.d/bsenv.sh \ + && bash /opt/buildserver/provision-apt-get-install https://deb.debian.org/debian \ + && tools=tools_r25.2.5-linux.zip \ + && mkdir -p /vagrant/cache \ + && curl https://dl.google.com/android/repository/$tools > /vagrant/cache/$tools \ + && echo "577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e /vagrant/cache/$tools" | sha256sum -c \ + && bash /opt/buildserver/provision-android-sdk \ + && bash /opt/buildserver/provision-android-ndk /opt/android-sdk/ndk \ + && bash /opt/buildserver/provision-gradle \ + && bash /opt/buildserver/provision-buildserverid $GIT_REV_PARSE_HEAD \ + && rm -rf /vagrant/cache \ + && apt-get autoremove --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Vagrant sudo setup for compatibility +RUN echo 'vagrant ALL = NOPASSWD: ALL' > /etc/sudoers.d/vagrant \ + && chmod 440 /etc/sudoers.d/vagrant \ + && sed -i -e 's/Defaults.*requiretty/#&/' /etc/sudoers diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 16bc2305..140e800a 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -78,13 +78,15 @@ Vagrant.configure("2") do |config| owner: 'root', group: 'root', create: true end - config.vm.provision "shell", path: "setup-env-vars", + config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars", args: ["/opt/android-sdk"] - config.vm.provision "shell", path: "provision-apt-get-install", + config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install", args: [configfile['debian_mirror']] - config.vm.provision "shell", path: "provision-android-sdk" - config.vm.provision "shell", path: "provision-android-ndk", + config.vm.provision "shell", name: "android-sdk", path: "provision-android-sdk" + config.vm.provision "shell", name: "android-ndk", path: "provision-android-ndk", args: ["/opt/android-sdk/ndk", "r21e", "r22b"] - config.vm.provision "shell", path: "provision-gradle" + config.vm.provision "shell", name: "gradle", path: "provision-gradle" + config.vm.provision "shell", name: "buildserverid", path: "provision-buildserverid", + args: [`git rev-parse HEAD`] end diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index 9672b309..ca2cc325 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -27,4 +27,4 @@ chmod g+w $NDK_BASE # ensure all users can read and execute the NDK chmod -R a+rX $NDK_BASE/ -find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x +find $NDK_BASE/ -type f -executable -exec chmod a+x -- {} + diff --git a/buildserver/provision-buildserverid b/buildserver/provision-buildserverid new file mode 100644 index 00000000..f5010c39 --- /dev/null +++ b/buildserver/provision-buildserverid @@ -0,0 +1,9 @@ +#!/bin/bash -e + +test -n "$1" + +echo "Writing buildserver ID ...ID is $1" +set -x +echo "$1" > /home/vagrant/buildserverid +# sync data before we halt() the machine, we had an empty buildserverid otherwise +sync diff --git a/buildserver/setup-env-vars b/buildserver/setup-env-vars index a4d96781..a5f53fd8 100644 --- a/buildserver/setup-env-vars +++ b/buildserver/setup-env-vars @@ -12,6 +12,8 @@ echo "# generated on "`date` > $bsenv echo export ANDROID_HOME=$1 >> $bsenv echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv echo "export DEBIAN_FRONTEND=noninteractive" >> $bsenv +echo 'export home_vagrant=/home/vagrant' >> $bsenv +echo 'export fdroidserver=$home_vagrant/fdroidserver' >> $bsenv chmod 0644 $bsenv diff --git a/makebuildserver b/makebuildserver index daafd4d4..24fe201a 100755 --- a/makebuildserver +++ b/makebuildserver @@ -584,15 +584,6 @@ def main(): run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock']) run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/']) - p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, - universal_newlines=True) - buildserverid = p.communicate()[0].strip() - logging.info("Writing buildserver ID ...ID is %s", buildserverid) - # sync data before we halt() the machine, we had an empty buildserverid otherwise - write_bsid_cmd = 'sh -c "echo \'{}\' >/home/vagrant/buildserverid; sync"'.format(buildserverid) - run_via_vagrant_ssh(v, write_bsid_cmd) - logging.debug("+ {}".format(write_bsid_cmd)) - logging.info("Stopping build server VM") v.halt()