Merge branch 'upgrade_buildserver_vm' into 'master'

Upgrade Buildserver VM to latest Debian (Bullseye) See merge request fdroid/fdroidserver!1012
2024-10-03 17:50:11 +02:00 · 2022-10-11 07:38:44 +00:00 · 2022-10-11 07:38:44 +00:00 · b0124aca1a
commit b0124aca1a
parent 7accb96b9e 4e839ac63b
13 changed files with 71 additions and 177 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -407,7 +407,7 @@ fdroid build:
 # fdroiddata because that one is known to work, and this is a CI job,
 # so it should be isolated from the normal churn of fdroiddata.
 plugin_fetchsrclibs:
-  image: debian:buster
+  image: debian:bullseye
  <<: *apt-template
  only:
    changes:
@ -430,7 +430,8 @@ plugin_fetchsrclibs:
    - . env/bin/activate
    - export PATH="$CI_PROJECT_DIR:$PATH"
    - export PYTHONPATH="$CI_PROJECT_DIR/examples"
-    - $pip install wheel  # to make this go away: "error: invalid command 'bdist_wheel'"
+    # workaround https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252
+    - export SETUPTOOLS_USE_DISTUTILS=stdlib
    - $pip install -e .
    - fdroid | grep fetchsrclibs

@ -588,7 +589,7 @@ docker:
    - cd buildserver
    - docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) .
    - docker tag $TEST_IMAGE $RELEASE_IMAGE
-    - docker tag $TEST_IMAGE ${RELEASE_IMAGE}-stretch
+    - docker tag $TEST_IMAGE ${RELEASE_IMAGE}-bullseye
    - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com
    # This avoids filling up gitlab.com free tier accounts with unused docker images.
    - if test -n "$FDROID_PUSH_DOCKER_IMAGE"; then
@ -598,4 +599,4 @@ docker:
          exit 0;
      fi
    - docker push $RELEASE_IMAGE
-    - docker push $RELEASE_IMAGE-stretch
+    - docker push $RELEASE_IMAGE-bullseye
--- a/buildserver/Dockerfile
+++ b/buildserver/Dockerfile
@ -1,5 +1,5 @@

-FROM debian:stretch
+FROM debian:bullseye

 ENV LANG=C.UTF-8 \
    DEBIAN_FRONTEND=noninteractive
--- a/buildserver/Vagrantfile
+++ b/buildserver/Vagrantfile
@ -77,6 +77,8 @@ Vagrant.configure("2") do |config|
      owner: 'root', group: 'root', create: true
  end

+  config.vm.provision "shell", name: "vagrant-insecure-private-key",
+    path: "provision-vagrant-insecure-private-key"
  config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars",
    args: ["/opt/android-sdk"]
  config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install",
--- a/buildserver/config.buildserver.yml
+++ b/buildserver/config.buildserver.yml
@ -1,6 +1,2 @@
 sdk_path: /opt/android-sdk
-
-java_paths:
-  8: /usr/lib/jvm/java-8-openjdk-amd64
-
 gradle_version_dir: /opt/gradle/versions
--- a/buildserver/provision-android-sdk
+++ b/buildserver/provision-android-sdk
@ -122,11 +122,6 @@ cat <<EOF > $ANDROID_HOME/licenses/intel-android-extra-license
 d975f751698a77b662f1254ddbeed3901e976f5a
 EOF

-echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1"
-echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1"
-echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2"
-echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.2"
-
 chmod a+X $(dirname $ANDROID_HOME/)
 chmod -R a+rX $ANDROID_HOME/
 chgrp vagrant $ANDROID_HOME
--- a/buildserver/provision-apt-get-install
+++ b/buildserver/provision-apt-get-install
@ -33,20 +33,15 @@ EOF

 if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then
    apt-get update || apt-get update
-    apt-get install apt-transport-https ca-certificates
+    apt-get install ca-certificates
 fi

 cat << EOF > /etc/apt/sources.list
-deb ${debian_mirror} stretch main
-deb http://security.debian.org/debian-security stretch/updates main
-deb ${debian_mirror} stretch-updates main
+deb ${debian_mirror} bullseye main
+deb https://security.debian.org/debian-security bullseye-security main
+deb ${debian_mirror} bullseye-updates main
 EOF
-echo "deb ${debian_mirror} stretch-backports main" > /etc/apt/sources.list.d/stretch-backports.list
-echo "deb ${debian_mirror} stretch-backports-sloppy main" > /etc/apt/sources.list.d/stretch-backports-sloppy.list
-echo "deb ${debian_mirror} testing main"           > /etc/apt/sources.list.d/testing.list
-printf "Package: *\nPin: release o=Debian,a=testing\nPin-Priority: -300\n" > /etc/apt/preferences.d/debian-testing
-
-dpkg --add-architecture i386
+echo "deb ${debian_mirror} bullseye-backports main" > /etc/apt/sources.list.d/backports.list

 apt-get update || apt-get update
 apt-get upgrade --download-only
@ -56,94 +51,19 @@ apt-get upgrade
 apt-get update || apt-get update

 packages="
- androguard/stretch-backports
- ant
- asn1c
- ant-contrib
- autoconf
- autoconf2.13
- automake
- automake1.11
- autopoint
- bison
- bzr
- ca-certificates-java
- cmake
- curl
+ default-jdk-headless
+ default-jre-headless
 dexdump
- disorderfs
- expect
- faketime
- flex
- gettext
- gettext-base
- git-core
- git-svn
- gperf
- gpg/stretch-backports-sloppy
- gpgconf/stretch-backports-sloppy
- libassuan0/stretch-backports
- libgpg-error0/stretch-backports
- javacc
- libarchive-zip-perl
- libexpat1-dev
- libgcc1:i386
- libglib2.0-dev
- liblzma-dev
- libncurses5:i386
- librsvg2-bin
- libsaxonb-java
- libssl-dev
- libstdc++6:i386
- libtool
- libtool-bin
- make
- maven
- mercurial
- nasm
- openjdk-8-jre-headless
- openjdk-8-jdk-headless
- optipng
- pkg-config
- python-gnupg
- python-lxml
- python-magic
- python-pip
- python-setuptools
- python3-asn1crypto/stretch-backports
- python3-defusedxml
- python3-git
- python3-gitdb
- python3-gnupg
- python3-pip
- python3-pyasn1
- python3-pyasn1-modules
- python3-qrcode
- python3-requests
- python3-setuptools
- python3-smmap
- python3-yaml
- python3-ruamel.yaml
- python3-pil
- python3-paramiko
- quilt
+ fdroidserver
+ gnupg
+ patch
 rsync
- scons
- sqlite3
- subversion
 sudo
- swig
- unzip
- xsltproc
- yasm
- zip
- zlib1g:i386
 "
+
 apt-get install $packages --download-only
 apt-get install $packages
+apt-get purge fdroidserver

 highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1`
 update-java-alternatives --set $highestjava
-
-# configure headless openjdk to work without gtk accessability dependencies
-sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-8-openjdk/accessibility.properties
--- a/buildserver/provision-vagrant-insecure-private-key
+++ b/buildserver/provision-vagrant-insecure-private-key
@ -0,0 +1,17 @@
+#!/bin/bash -e
+#
+# Vagrant uses the "insecure private key" to establish the first SSH
+# connection to a new VM based on a clean public box.  In theory, the
+# `vagrant package` command should do that automatically.  This
+# process is still using custom code instead of `vagrant package`,
+# hence this script.
+#
+# https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/issues/12
+# https://www.vagrantup.com/docs/vagrantfile/ssh_settings#config-ssh-private_key_path
+#
+# This public key can be generated using:
+#   ssh-keygen -y -f  ~/.vagrant.d/insecure_private_key
+
+echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== ~/.vagrant.d/insecure_private_key" > /home/vagrant/.ssh/authorized_keys
+chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
+chmod 600 /home/vagrant/.ssh/authorized_keys
--- a/buildserver/setup-env-vars
+++ b/buildserver/setup-env-vars
@ -18,5 +18,9 @@ echo 'export fdroidserver=$home_vagrant/fdroidserver' >> $bsenv
 chmod 0644 $bsenv

 # make sure that SSH never hangs at a password or key prompt
-printf '    StrictHostKeyChecking yes\n' >> /etc/ssh/ssh_config
-printf '    BatchMode yes\n' >> /etc/ssh/ssh_config
+mkdir -p /etc/ssh/ssh_config.d/
+cat << EOF >> /etc/ssh/ssh_config.d/fdroid
+Host *
+    StrictHostKeyChecking yes
+    BatchMode yes
+EOF
--- a/fdroidserver/import_subcommand.py
+++ b/fdroidserver/import_subcommand.py
@ -334,7 +334,7 @@ def main():
    package_json = tmp_importer_dir / 'package.json'  # react-native
    pubspec_yaml = tmp_importer_dir / 'pubspec.yaml'  # flutter
    if package_json.exists():
-        build.sudo = ['apt-get update || apt-get update', 'apt-get install -t stretch-backports npm', 'npm install -g react-native-cli']
+        build.sudo = ['apt-get update || apt-get update', 'apt-get install npm', 'npm install -g react-native-cli']
        build.init = ['npm install']
        with package_json.open() as fp:
            data = json.load(fp)
--- a/fdroidserver/vmtools.py
+++ b/fdroidserver/vmtools.py
@ -465,8 +465,6 @@ class LibvirtBuildVm(FDroidBuildVm):
            vagrantfile = textwrap.dedent(
                """\
                  Vagrant.configure("2") do |config|
-                    config.ssh.username = "vagrant"
-                    config.ssh.password = "vagrant"

                    config.vm.provider :libvirt do |libvirt|

--- a/2
+++ b/2
@ -62,7 +62,7 @@ vagrant global-status \
 # so we need to "manually" clone the git repo here…
 cd $WORKSPACE

-# set up Android SDK to use the Debian packages in stretch
+# set up Android SDK to use the Debian packages
 export ANDROID_HOME=/usr/lib/android-sdk

 # now build the whole archive
--- a/2
+++ b/2
@ -29,7 +29,7 @@ fi
 set -e
 set -x

-# set up Android SDK to use the Debian packages in stretch
+# set up Android SDK to use the Debian packages
 export ANDROID_HOME=/usr/lib/android-sdk

 rm -rf "$WORKSPACE/.testfiles"
--- a/89
+++ b/89
@ -43,74 +43,35 @@ logging.basicConfig(format=logformat, level=loglevel)

 tail = None

-BASEBOX_DEFAULT = 'fdroid/basebox-stretch64'
-BASEBOX_VERSION_DEFAULT = '0.5.1'
+BASEBOX_DEFAULT = 'fdroid/bullseye64'
+BASEBOX_VERSION_DEFAULT = "11.20221010.1"
 BASEBOX_CHECKSUMS = {
-    '0.6.1': {
-        'libvirt': {
-            'box.img': 'a45342ad238271815f9197325cd0317d6df87e1c20372015532919d817d0dc07',
-            'metadata.json': '9b5f62362ce3cd25c50881d8ae124879fc21ed4fdb16cc78d57058f116680f25',
-            'Vagrantfile': '4435901624f21dad201c3bd7f0d8d4ece842bc9fbbb70e312eee54f07173f24e',
+    "11.20221010.1": {
+        "libvirt": {
+            "box.img": "c2114aa276c176fa65b8072f5dcd1e8a6ab9f7d15fd5da791727a0164fd43254",
+            "Vagrantfile": "f9c6fcbb47a4d0d33eb066859c8e87efd642287a638bd7da69a9e7a6f25fec47",
+            "metadata.json": "42b96a01106c25f3a222ddad0baead0b811cc64926f924fb836bbfa43580e646",
        },
-        'virtualbox': {
-            'box-disk1.vmdk': '6b536f26dcee137aca9a3f5f6f20aef795193ef2e8c387a0ffbdb7c5fe2ec0fb',
-            'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
-            'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
-            'Vagrantfile': '95c64a0e82a6420845c05038c4c97b3aba629b09eb2b78e879423d06f6b54a54',
+        "virtualbox": {
+            "box.ovf": "5e4de5f1f4b481b2c1917c0b2f6e6334f4741cc18c5b278e3bafb094535ff2cb",
+            "box.vmdk": "737053bc886037ae76bb38a1776eba2a5579d49423de990e93ef4a3f0cab4f1c",
+            "Vagrantfile": "0bbc2ae97668d8da27ab97b766752dcd0bf9e41900e21057de15a58ee7fae47d",
+            "metadata.json": "ffdaa989f2f6932cd8042e1102371f405cc7ad38e324210a1326192e4689e83a",
        }
    },
-    '0.6.0': {
+    '11.20220317.1': {
        'libvirt': {
-            'box.img': '82c2c3548cf48f0f4c6601f40f8bec36ff37e9a74d6f717067a526250ad790ad',
-            'metadata.json': '9b5f62362ce3cd25c50881d8ae124879fc21ed4fdb16cc78d57058f116680f25',
-            'Vagrantfile': '4435901624f21dad201c3bd7f0d8d4ece842bc9fbbb70e312eee54f07173f24e',
+            'box.img': 'fbde152a2f61d191983be9d1dbeae2591af32cca1ec27daa342485d97187515e',
+            'metadata.json': '42b96a01106c25f3a222ddad0baead0b811cc64926f924fb836bbfa43580e646',
+            'Vagrantfile': 'f9c6fcbb47a4d0d33eb066859c8e87efd642287a638bd7da69a9e7a6f25fec47',
        },
        'virtualbox': {
-            'box-disk1.vmdk': '6b536f26dcee137aca9a3f5f6f20aef795193ef2e8c387a0ffbdb7c5fe2ec0fb',
-            'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
-            'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
-            'Vagrantfile': '95c64a0e82a6420845c05038c4c97b3aba629b09eb2b78e879423d06f6b54a54',
+            'box.ovf': 'becd5cea2666d42e12def13a91766aa0d4b0e8e6f53102486c2a6cdb4e401b08',
+            'box.vmdk': '49c96a58a3ee99681d348075864a290c60a8d334fddd21be453c825fcee75eda',
+            'metadata.json': 'ffdaa989f2f6932cd8042e1102371f405cc7ad38e324210a1326192e4689e83a',
+            'Vagrantfile': '0bbc2ae97668d8da27ab97b766752dcd0bf9e41900e21057de15a58ee7fae47d',
        }
    },
-    '0.5.1': {
-        'libvirt': {
-            'box.img': 'ad015940b866e36a593ef5fa0035ec6703f74a7f082ab76a1d2bd9463714cd4a',
-            'metadata.json': '5ced8ecf886722a5152095e099b778b1d2b859c2e1dcf834182274034b8a629d',
-            'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
-        },
-        'virtualbox': {
-            'box-disk1.vmdk': 'cba36a9c9814bdff9aabaea8786c27477ef8958cf6ee65ad844cb2726bdab93e',
-            'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
-            'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
-            'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
-        }
-    },
-    '0.5': {
-        'virtualbox': {
-            'box-disk1.vmdk': '8834d5eb78758437c2517f83282172fd5e3842d88f657d577592d0917cd02f89',
-            'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875',
-            'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
-            'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
-        },
-        'libvirt': {
-            'box.img': '2ef5f1fdc98c24a4f67cecb526d21e1d73dedf5a0072ceff528a0e75da3ff452',
-            'metadata.json': 'da79a5e2327dcf81a18a9d66a6e91205a20e440f23d3928e633fd39d60c641e5',
-            'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
-        }
-    },
-    '0.3': {
-        'libvirt': {
-            'box.img': '24f06f415dde4cdb01d68c904fc57386ea060ba7b94e700670c58694b3d3635e',
-            'metadata.json': '0965955659082fd2e67723deb3311ba253c96153d3176d856db1b3e6e461cf23',
-            'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260',
-        },
-        'virtualbox': {
-            'box-disk1.vmdk': '103114977f1a36f7121ef9b3a1495129baa10bfedfada61a13345c8863c4dcd6',
-            'box.ovf': '33a5fbaf3dba443237baefcba6d56ca7a76121ca530f1140aa8263a69d7d3695',
-            'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99',
-            'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14',
-        }
-    }
 }

 config = {
@ -138,7 +99,7 @@ logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, inden
 if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config:
    config['basebox_version'] = BASEBOX_VERSION_DEFAULT
 # note: vagrant allows putting '/' into the name of a local box,
-# so this check is not completely relyable, but better than nothing
+# so this check is not completely reliable, but better than nothing
 if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']:
    logging.critical("Can not get version '{version}' for basebox '{box}', "
                     "vagrant does not support versioning for locally added boxes."
@ -468,7 +429,7 @@ def main():
        logging.info('Halting %s', serverdir)
        v.halt()
        with open(vf, 'r', encoding='utf-8') as f:
-            oldconfig = yaml.load(f)
+            oldconfig = yaml.safe_load(f)
        if config != oldconfig:
            logging.info("Server configuration has changed, rebuild from scratch is required")
            vm.destroy()
@ -479,7 +440,7 @@ def main():
        logging.info("No existing server - building from scratch")
    if writevf:
        with open(vf, 'w', encoding='utf-8') as f:
-            yaml.dump(config, f)
+            yaml.safe_dump(config, f)

    # Check if selected provider is supported
    if config['vm_provider'] not in ['libvirt', 'virtualbox']:
@ -488,7 +449,7 @@ def main():
                         "virtualbox, libvirt)"
                         .format(vm_provider=config['cm_provider']))
        sys.exit(1)
-    # Check if selected basebox is available
+    # Check if selected Vagrant box is available
    available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']]
    if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider:
        logging.critical("Vagrant box '{basebox}' not available "
@ -501,7 +462,7 @@ def main():
                                 basebox=config['basebox']))
        sys.exit(1)

-    # download and verfiy fdroid pre-built basebox
+    # Download and verify pre-built Vagrant boxes
    if config['basebox'] == BASEBOX_DEFAULT:
        buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default'])
        if buildserver_not_created or options.clean: