Merge branch 'basic-buildserver-docker-image' into 'master'

basic buildserver docker image

See merge request fdroid/fdroidserver!1069

.gitlab-ci.yml

@@ -395,8 +395,10 @@ plugin_fetchsrclibs:
         python3-cffi
         python3-cryptography
         python3-matplotlib
+        python3-nacl
         python3-pip
         python3-pil
+        python3-pycparser
         python3-venv
     - python3 -m venv --system-site-packages env
     - . env/bin/activate
@@ -529,3 +531,25 @@ pages:
   needs: ["Build documentation"]
   rules:
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'  # only publish pages on default (master) branch
+
+
+docker:
+  dependencies:
+    - fdroid build
+  only:
+    changes:
+      - .gitlab-ci.yml
+      - makebuildserver
+      - buildserver/*
+  image: docker:git
+  services:
+    - docker:dind
+  variables:
+    TEST_IMAGE: registry.gitlab.com/$CI_PROJECT_NAMESPACE/${CI_PROJECT_NAME}:$CI_BUILD_REF_NAME
+    RELEASE_IMAGE: registry.gitlab.com/$CI_PROJECT_NAMESPACE/${CI_PROJECT_NAME}:buildserver
+  script:
+    - cd buildserver
+    - docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) .
+    - docker tag $TEST_IMAGE $RELEASE_IMAGE
+    - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com
+    - docker push $RELEASE_IMAGE

buildserver/Dockerfile

@@ -0,0 +1,59 @@
+
+FROM debian:stretch
+
+ENV LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive
+
+RUN echo Etc/UTC > /etc/timezone \
+	&& echo 'APT::Install-Recommends "0";' \
+		'APT::Install-Suggests "0";' \
+		'APT::Acquire::Retries "20";' \
+		'APT::Get::Assume-Yes "true";' \
+		'Dpkg::Use-Pty "0";' \
+		'quiet "1";' \
+        >> /etc/apt/apt.conf.d/99gitlab
+
+# provision-apt-proxy was deliberately omitted, its not relevant in Docker
+COPY 	provision-android-ndk \
+	provision-android-sdk \
+	provision-apt-get-install \
+	provision-buildserverid \
+	provision-gradle \
+	setup-env-vars \
+	/opt/buildserver/
+
+ARG GIT_REV_PARSE_HEAD=unspecified
+LABEL org.opencontainers.image.revision=$GIT_REV_PARSE_HEAD
+
+# setup 'vagrant' user for compatibility
+RUN useradd --create-home -s /bin/bash vagrant && echo -n 'vagrant:vagrant' | chpasswd
+
+# the provision scripts must be run in the same order as in Vagrantfile
+# - vagrant needs openssh-client iproute2 ssh sudo
+# - ansible needs python3
+RUN printf "path-exclude=/usr/share/locale/*\npath-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-include=/usr/share/doc/*/copyright\n" >/etc/dpkg/dpkg.cfg.d/01_nodoc \
+	&& mkdir -p /usr/share/man/man1 \
+	&& apt-get update \
+	&& apt-get upgrade \
+	&& apt-get dist-upgrade \
+	&& apt-get install openssh-client iproute2 python3 openssh-server sudo \
+	&& bash /opt/buildserver/setup-env-vars /opt/android-sdk \
+	&& . /etc/profile.d/bsenv.sh \
+	&& bash /opt/buildserver/provision-apt-get-install https://deb.debian.org/debian \
+	&& tools=tools_r25.2.5-linux.zip \
+	&& mkdir -p /vagrant/cache \
+	&& curl https://dl.google.com/android/repository/$tools > /vagrant/cache/$tools \
+	&& echo "577516819c8b5fae680f049d39014ff1ba4af870b687cab10595783e6f22d33e /vagrant/cache/$tools" | sha256sum -c \
+	&& bash /opt/buildserver/provision-android-sdk \
+	&& bash /opt/buildserver/provision-android-ndk /opt/android-sdk/ndk \
+	&& bash /opt/buildserver/provision-gradle \
+	&& bash /opt/buildserver/provision-buildserverid $GIT_REV_PARSE_HEAD \
+	&& rm -rf /vagrant/cache \
+	&& apt-get autoremove --purge \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/*
+
+# Vagrant sudo setup for compatibility
+RUN echo 'vagrant ALL = NOPASSWD: ALL' > /etc/sudoers.d/vagrant \
+	&& chmod 440 /etc/sudoers.d/vagrant \
+	&& sed -i -e 's/Defaults.*requiretty/#&/' /etc/sudoers

buildserver/Vagrantfile

@@ -78,13 +78,15 @@ Vagrant.configure("2") do |config|
       owner: 'root', group: 'root', create: true
   end
 
-  config.vm.provision "shell", path: "setup-env-vars",
+  config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars",
     args: ["/opt/android-sdk"]
-  config.vm.provision "shell", path: "provision-apt-get-install",
+  config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install",
     args: [configfile['debian_mirror']]
-  config.vm.provision "shell", path: "provision-android-sdk"
+  config.vm.provision "shell", name: "android-sdk", path: "provision-android-sdk"
-  config.vm.provision "shell", path: "provision-android-ndk",
+  config.vm.provision "shell", name: "android-ndk", path: "provision-android-ndk",
     args: ["/opt/android-sdk/ndk", "r21e", "r22b"]
-  config.vm.provision "shell", path: "provision-gradle"
+  config.vm.provision "shell", name: "gradle", path: "provision-gradle"
+  config.vm.provision "shell", name: "buildserverid", path: "provision-buildserverid",
+    args: [`git rev-parse HEAD`]
 
 end

buildserver/provision-android-ndk

@@ -27,4 +27,4 @@ chmod g+w $NDK_BASE
 
 # ensure all users can read and execute the NDK
 chmod -R a+rX $NDK_BASE/
-find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x
+find $NDK_BASE/ -type f -executable -exec chmod a+x -- {} +

buildserver/provision-buildserverid

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+test -n "$1"
+
+echo "Writing buildserver ID ...ID is $1"
+set -x
+echo "$1" > /home/vagrant/buildserverid
+# sync data before we halt() the machine, we had an empty buildserverid otherwise
+sync

buildserver/setup-env-vars

@@ -12,6 +12,8 @@ echo "# generated on "`date` > $bsenv
 echo export ANDROID_HOME=$1 >> $bsenv
 echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv
 echo "export DEBIAN_FRONTEND=noninteractive" >> $bsenv
+echo 'export home_vagrant=/home/vagrant' >> $bsenv
+echo 'export fdroidserver=$home_vagrant/fdroidserver' >> $bsenv
 
 chmod 0644 $bsenv
 

makebuildserver

@@ -584,15 +584,6 @@ def main():
         run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
         run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
 
-    p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
-                         universal_newlines=True)
-    buildserverid = p.communicate()[0].strip()
-    logging.info("Writing buildserver ID ...ID is %s", buildserverid)
-    # sync data before we halt() the machine, we had an empty buildserverid otherwise
-    write_bsid_cmd = 'sh -c "echo \'{}\' >/home/vagrant/buildserverid; sync"'.format(buildserverid)
-    run_via_vagrant_ssh(v, write_bsid_cmd)
-    logging.debug("+ {}".format(write_bsid_cmd))
-
     logging.info("Stopping build server VM")
     v.halt()