From df46eb86c339c10826a111b6ec151ba0eff81a48 Mon Sep 17 00:00:00 2001 From: FestplattenSchnitzel Date: Fri, 3 Sep 2021 12:18:13 +0200 Subject: [PATCH 01/15] Upgrade Buildserver VM Use Vagrant boxes built with cloud-team/debian-vagrant-images instead of fdroid/basebox, Use Debian Bullseye (11) instead of Debian Stretch (9) --- .gitlab-ci.yml | 4 +- buildserver/Dockerfile | 2 +- buildserver/provision-apt-get-install | 23 ++++---- fdroidserver/import_subcommand.py | 2 +- jenkins-build-all | 2 +- jenkins-test | 2 +- makebuildserver | 78 +++++---------------------- 7 files changed, 29 insertions(+), 84 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4c9a813..7bfe690d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -588,7 +588,7 @@ docker: - cd buildserver - docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) . - docker tag $TEST_IMAGE $RELEASE_IMAGE - - docker tag $TEST_IMAGE ${RELEASE_IMAGE}-stretch + - docker tag $TEST_IMAGE ${RELEASE_IMAGE}-bullseye - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com # This avoids filling up gitlab.com free tier accounts with unused docker images. - if test -n "$FDROID_PUSH_DOCKER_IMAGE"; then @@ -598,4 +598,4 @@ docker: exit 0; fi - docker push $RELEASE_IMAGE - - docker push $RELEASE_IMAGE-stretch + - docker push $RELEASE_IMAGE-bullseye diff --git a/buildserver/Dockerfile b/buildserver/Dockerfile index 3d9ee52f..6ecdf544 100644 --- a/buildserver/Dockerfile +++ b/buildserver/Dockerfile @@ -1,5 +1,5 @@ -FROM debian:stretch +FROM debian:bullseye ENV LANG=C.UTF-8 \ DEBIAN_FRONTEND=noninteractive diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 24cb1a5e..17b229fd 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -37,14 +37,11 @@ if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then fi cat << EOF > /etc/apt/sources.list -deb ${debian_mirror} stretch main -deb http://security.debian.org/debian-security stretch/updates main -deb ${debian_mirror} stretch-updates main +deb ${debian_mirror} bullseye main +deb http://security.debian.org/debian-security bullseye-security main +deb ${debian_mirror} bullseye-updates main EOF -echo "deb ${debian_mirror} stretch-backports main" > /etc/apt/sources.list.d/stretch-backports.list -echo "deb ${debian_mirror} stretch-backports-sloppy main" > /etc/apt/sources.list.d/stretch-backports-sloppy.list -echo "deb ${debian_mirror} testing main" > /etc/apt/sources.list.d/testing.list -printf "Package: *\nPin: release o=Debian,a=testing\nPin-Priority: -300\n" > /etc/apt/preferences.d/debian-testing +echo "deb ${debian_mirror} bullseye-backports main" > /etc/apt/sources.list.d/backports.list dpkg --add-architecture i386 @@ -56,7 +53,7 @@ apt-get upgrade apt-get update || apt-get update packages=" - androguard/stretch-backports + androguard ant asn1c ant-contrib @@ -80,10 +77,10 @@ packages=" git-core git-svn gperf - gpg/stretch-backports-sloppy - gpgconf/stretch-backports-sloppy - libassuan0/stretch-backports - libgpg-error0/stretch-backports + gpg + gpgconf + libassuan0 + libgpg-error0 javacc libarchive-zip-perl libexpat1-dev @@ -110,7 +107,7 @@ packages=" python-magic python-pip python-setuptools - python3-asn1crypto/stretch-backports + python3-asn1crypto python3-defusedxml python3-git python3-gitdb diff --git a/fdroidserver/import_subcommand.py b/fdroidserver/import_subcommand.py index ec3b0036..8f89148b 100644 --- a/fdroidserver/import_subcommand.py +++ b/fdroidserver/import_subcommand.py @@ -334,7 +334,7 @@ def main(): package_json = tmp_importer_dir / 'package.json' # react-native pubspec_yaml = tmp_importer_dir / 'pubspec.yaml' # flutter if package_json.exists(): - build.sudo = ['apt-get update || apt-get update', 'apt-get install -t stretch-backports npm', 'npm install -g react-native-cli'] + build.sudo = ['apt-get update || apt-get update', 'apt-get install npm', 'npm install -g react-native-cli'] build.init = ['npm install'] with package_json.open() as fp: data = json.load(fp) diff --git a/jenkins-build-all b/jenkins-build-all index aa663fc5..5dfebbf2 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -62,7 +62,7 @@ vagrant global-status \ # so we need to "manually" clone the git repo here… cd $WORKSPACE -# set up Android SDK to use the Debian packages in stretch +# set up Android SDK to use the Debian packages export ANDROID_HOME=/usr/lib/android-sdk # now build the whole archive diff --git a/jenkins-test b/jenkins-test index fb8eb3b6..cdacea32 100755 --- a/jenkins-test +++ b/jenkins-test @@ -29,7 +29,7 @@ fi set -e set -x -# set up Android SDK to use the Debian packages in stretch +# set up Android SDK to use the Debian packages export ANDROID_HOME=/usr/lib/android-sdk rm -rf "$WORKSPACE/.testfiles" diff --git a/makebuildserver b/makebuildserver index cb4d1b92..3aa93765 100755 --- a/makebuildserver +++ b/makebuildserver @@ -43,74 +43,22 @@ logging.basicConfig(format=logformat, level=loglevel) tail = None -BASEBOX_DEFAULT = 'fdroid/basebox-stretch64' -BASEBOX_VERSION_DEFAULT = '0.5.1' +BASEBOX_DEFAULT = 'fdroid/bullseye64' +BASEBOX_VERSION_DEFAULT = '11.20220317.1' BASEBOX_CHECKSUMS = { - '0.6.1': { + '11.20220317.1': { 'libvirt': { - 'box.img': 'a45342ad238271815f9197325cd0317d6df87e1c20372015532919d817d0dc07', - 'metadata.json': '9b5f62362ce3cd25c50881d8ae124879fc21ed4fdb16cc78d57058f116680f25', - 'Vagrantfile': '4435901624f21dad201c3bd7f0d8d4ece842bc9fbbb70e312eee54f07173f24e', + 'box.img': 'fbde152a2f61d191983be9d1dbeae2591af32cca1ec27daa342485d97187515e', + 'metadata.json': '42b96a01106c25f3a222ddad0baead0b811cc64926f924fb836bbfa43580e646', + 'Vagrantfile': 'f9c6fcbb47a4d0d33eb066859c8e87efd642287a638bd7da69a9e7a6f25fec47', }, 'virtualbox': { - 'box-disk1.vmdk': '6b536f26dcee137aca9a3f5f6f20aef795193ef2e8c387a0ffbdb7c5fe2ec0fb', - 'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875', - 'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99', - 'Vagrantfile': '95c64a0e82a6420845c05038c4c97b3aba629b09eb2b78e879423d06f6b54a54', + 'box.ovf': 'becd5cea2666d42e12def13a91766aa0d4b0e8e6f53102486c2a6cdb4e401b08', + 'box.vmdk': '49c96a58a3ee99681d348075864a290c60a8d334fddd21be453c825fcee75eda', + 'metadata.json': 'ffdaa989f2f6932cd8042e1102371f405cc7ad38e324210a1326192e4689e83a', + 'Vagrantfile': '0bbc2ae97668d8da27ab97b766752dcd0bf9e41900e21057de15a58ee7fae47d', } }, - '0.6.0': { - 'libvirt': { - 'box.img': '82c2c3548cf48f0f4c6601f40f8bec36ff37e9a74d6f717067a526250ad790ad', - 'metadata.json': '9b5f62362ce3cd25c50881d8ae124879fc21ed4fdb16cc78d57058f116680f25', - 'Vagrantfile': '4435901624f21dad201c3bd7f0d8d4ece842bc9fbbb70e312eee54f07173f24e', - }, - 'virtualbox': { - 'box-disk1.vmdk': '6b536f26dcee137aca9a3f5f6f20aef795193ef2e8c387a0ffbdb7c5fe2ec0fb', - 'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875', - 'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99', - 'Vagrantfile': '95c64a0e82a6420845c05038c4c97b3aba629b09eb2b78e879423d06f6b54a54', - } - }, - '0.5.1': { - 'libvirt': { - 'box.img': 'ad015940b866e36a593ef5fa0035ec6703f74a7f082ab76a1d2bd9463714cd4a', - 'metadata.json': '5ced8ecf886722a5152095e099b778b1d2b859c2e1dcf834182274034b8a629d', - 'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260', - }, - 'virtualbox': { - 'box-disk1.vmdk': 'cba36a9c9814bdff9aabaea8786c27477ef8958cf6ee65ad844cb2726bdab93e', - 'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875', - 'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99', - 'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14', - } - }, - '0.5': { - 'virtualbox': { - 'box-disk1.vmdk': '8834d5eb78758437c2517f83282172fd5e3842d88f657d577592d0917cd02f89', - 'box.ovf': 'cbdd6315187d4ce8ff15ed5a00a2c8b0d33abe6b0356439ce4d8d9ac3724f875', - 'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99', - 'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14', - }, - 'libvirt': { - 'box.img': '2ef5f1fdc98c24a4f67cecb526d21e1d73dedf5a0072ceff528a0e75da3ff452', - 'metadata.json': 'da79a5e2327dcf81a18a9d66a6e91205a20e440f23d3928e633fd39d60c641e5', - 'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260', - } - }, - '0.3': { - 'libvirt': { - 'box.img': '24f06f415dde4cdb01d68c904fc57386ea060ba7b94e700670c58694b3d3635e', - 'metadata.json': '0965955659082fd2e67723deb3311ba253c96153d3176d856db1b3e6e461cf23', - 'Vagrantfile': 'cc7b8edb26481c158b2c28d15d32f7e146de892847c9308ac262678cf0ae8260', - }, - 'virtualbox': { - 'box-disk1.vmdk': '103114977f1a36f7121ef9b3a1495129baa10bfedfada61a13345c8863c4dcd6', - 'box.ovf': '33a5fbaf3dba443237baefcba6d56ca7a76121ca530f1140aa8263a69d7d3695', - 'metadata.json': '098439524f76cafe026140b787ca419297a055a3f6006b9d60e6d5326d18ba99', - 'Vagrantfile': 'ae50c3d152c3016e853176005d1a5da7a8e6ae424c9074e93b1a1015aa2f2e14', - } - } } config = { @@ -138,7 +86,7 @@ logging.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, inden if config['basebox'] == BASEBOX_DEFAULT and 'basebox_version' not in config: config['basebox_version'] = BASEBOX_VERSION_DEFAULT # note: vagrant allows putting '/' into the name of a local box, -# so this check is not completely relyable, but better than nothing +# so this check is not completely reliable, but better than nothing if 'basebox_version' in config and 'basebox' in config and '/' not in config['basebox']: logging.critical("Can not get version '{version}' for basebox '{box}', " "vagrant does not support versioning for locally added boxes." @@ -488,7 +436,7 @@ def main(): "virtualbox, libvirt)" .format(vm_provider=config['cm_provider'])) sys.exit(1) - # Check if selected basebox is available + # Check if selected Vagrant box is available available_boxes_by_provider = [x.name for x in v.box_list() if x.provider == config['vm_provider']] if '/' not in config['basebox'] and config['basebox'] not in available_boxes_by_provider: logging.critical("Vagrant box '{basebox}' not available " @@ -501,7 +449,7 @@ def main(): basebox=config['basebox'])) sys.exit(1) - # download and verfiy fdroid pre-built basebox + # Download and verify pre-built Vagrant boxes if config['basebox'] == BASEBOX_DEFAULT: buildserver_not_created = any([True for x in v.status() if x.state == 'not_created' and x.name == 'default']) if buildserver_not_created or options.clean: From cc641b5470c3fb3755c88b64d47c6e26db7aca27 Mon Sep 17 00:00:00 2001 From: FestplattenSchnitzel Date: Fri, 3 Sep 2021 13:09:55 +0200 Subject: [PATCH 02/15] Stop adding i386 architecture to dpkg --- buildserver/provision-apt-get-install | 2 -- 1 file changed, 2 deletions(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 17b229fd..96655b4b 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -43,8 +43,6 @@ deb ${debian_mirror} bullseye-updates main EOF echo "deb ${debian_mirror} bullseye-backports main" > /etc/apt/sources.list.d/backports.list -dpkg --add-architecture i386 - apt-get update || apt-get update apt-get upgrade --download-only apt-get upgrade From be5aefd3eebe6abc915c344c04f9b1d028f89dcd Mon Sep 17 00:00:00 2001 From: FestplattenSchnitzel Date: Fri, 3 Sep 2021 13:48:36 +0200 Subject: [PATCH 03/15] Install required packages for fdroidserver automatically --- buildserver/provision-apt-get-install | 90 +++------------------------ 1 file changed, 8 insertions(+), 82 deletions(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 96655b4b..634a01a5 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -51,94 +51,20 @@ apt-get upgrade apt-get update || apt-get update packages=" - androguard - ant - asn1c - ant-contrib - autoconf - autoconf2.13 - automake - automake1.11 - autopoint - bison - bzr - ca-certificates-java - cmake - curl + default-jdk-headless + default-jre-headless dexdump - disorderfs - expect - faketime - flex - gettext - gettext-base - git-core - git-svn - gperf - gpg - gpgconf - libassuan0 - libgpg-error0 - javacc - libarchive-zip-perl - libexpat1-dev - libgcc1:i386 - libglib2.0-dev - liblzma-dev - libncurses5:i386 - librsvg2-bin - libsaxonb-java - libssl-dev - libstdc++6:i386 - libtool - libtool-bin - make - maven - mercurial - nasm - openjdk-8-jre-headless - openjdk-8-jdk-headless - optipng - pkg-config - python-gnupg - python-lxml - python-magic - python-pip - python-setuptools - python3-asn1crypto - python3-defusedxml - python3-git - python3-gitdb - python3-gnupg - python3-pip - python3-pyasn1 - python3-pyasn1-modules - python3-qrcode - python3-requests - python3-setuptools - python3-smmap - python3-yaml - python3-ruamel.yaml - python3-pil - python3-paramiko - quilt - rsync - scons - sqlite3 - subversion + fdroidserver + gnupg sudo - swig - unzip - xsltproc - yasm - zip - zlib1g:i386 " + apt-get install $packages --download-only apt-get install $packages +apt-get purge fdroidserver highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1` update-java-alternatives --set $highestjava -# configure headless openjdk to work without gtk accessability dependencies -sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-8-openjdk/accessibility.properties +# configure headless openjdk to work without gtk accessibility dependencies +sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-11-openjdk/accessibility.properties From cb2c3a34bb57b9d020f2f62d47fe72d020fc5ccc Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 7 Sep 2021 11:10:44 +0000 Subject: [PATCH 04/15] remove hack to disable GNOME.Accessibility for minimal deps This is commented out by default on all OpenJDK packages since Java 8. https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1012?diff_id=263085985#note_726457435 root@sid:/# grep Accessibility /etc/java-*-openjdk/accessibility.properties /etc/java-11-openjdk/accessibility.properties:#assistive_technologies=org.GNOME.Accessibility.AtkWrapper /etc/java-17-openjdk/accessibility.properties:#assistive_technologies=org.GNOME.Accessibility.AtkWrapper /etc/java-18-openjdk/accessibility.properties:#assistive_technologies=org.GNOME.Accessibility.AtkWrapper /etc/java-8-openjdk/accessibility.properties:assistive_technologies=org.GNOME.Accessibility.AtkWrapper --- buildserver/provision-apt-get-install | 3 --- 1 file changed, 3 deletions(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 634a01a5..a1cfd6bc 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -65,6 +65,3 @@ apt-get purge fdroidserver highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1` update-java-alternatives --set $highestjava - -# configure headless openjdk to work without gtk accessibility dependencies -sed -i -e 's@\(assistive_technologies=org.GNOME.Accessibility.AtkWrapper\)@#\1@' /etc/java-11-openjdk/accessibility.properties From b4f8ab289dd458110fe16b64b1bbfe5fa05fa407 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sun, 17 Oct 2021 21:46:39 +0200 Subject: [PATCH 05/15] Use yaml.safe_* --- makebuildserver | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makebuildserver b/makebuildserver index 3aa93765..68d07547 100755 --- a/makebuildserver +++ b/makebuildserver @@ -416,7 +416,7 @@ def main(): logging.info('Halting %s', serverdir) v.halt() with open(vf, 'r', encoding='utf-8') as f: - oldconfig = yaml.load(f) + oldconfig = yaml.safe_load(f) if config != oldconfig: logging.info("Server configuration has changed, rebuild from scratch is required") vm.destroy() @@ -427,7 +427,7 @@ def main(): logging.info("No existing server - building from scratch") if writevf: with open(vf, 'w', encoding='utf-8') as f: - yaml.dump(config, f) + yaml.safe_dump(config, f) # Check if selected provider is supported if config['vm_provider'] not in ['libvirt', 'virtualbox']: From d762f423ce227fe67ac7f8e44739935b3bb4955e Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sun, 17 Oct 2021 21:47:21 +0200 Subject: [PATCH 06/15] Install patch and rsync (required by fdroidserver) --- buildserver/provision-apt-get-install | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index a1cfd6bc..916798aa 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -56,6 +56,8 @@ packages=" dexdump fdroidserver gnupg + patch + rsync sudo " From f03e812a137c99bd0a78ac0f1c8a5fc1ae8328b6 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sun, 17 Oct 2021 22:58:12 +0200 Subject: [PATCH 07/15] Drop obsolete apt-transport-https --- buildserver/provision-apt-get-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index 916798aa..b4855f51 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -33,7 +33,7 @@ EOF if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then apt-get update || apt-get update - apt-get install apt-transport-https ca-certificates + apt-get install ca-certificates fi cat << EOF > /etc/apt/sources.list From dbaa3f6ec2509c2556c3bb0bb872f60a5182fc85 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 17 Jan 2022 09:08:28 +0100 Subject: [PATCH 08/15] Stop installing ConstraintLayout sdkmanager does not work anymore with Java 11 and these can be installed in the app's build metadata. --- buildserver/provision-android-sdk | 5 ----- 1 file changed, 5 deletions(-) diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index e8fa1834..ac3a0b99 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -122,11 +122,6 @@ cat < $ANDROID_HOME/licenses/intel-android-extra-license d975f751698a77b662f1254ddbeed3901e976f5a EOF -echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1" -echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1" -echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2" -echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.2" - chmod a+X $(dirname $ANDROID_HOME/) chmod -R a+rX $ANDROID_HOME/ chgrp vagrant $ANDROID_HOME From 9ddf38b69abc990583ef093eaaa71c20e9b9b386 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 18 Oct 2021 12:14:21 +0200 Subject: [PATCH 09/15] Drop vagrant ssh config Not needed and breaks login with debian/bullseye64 image. --- fdroidserver/vmtools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index f4e3a416..9462e446 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -465,8 +465,6 @@ class LibvirtBuildVm(FDroidBuildVm): vagrantfile = textwrap.dedent( """\ Vagrant.configure("2") do |config| - config.ssh.username = "vagrant" - config.ssh.password = "vagrant" config.vm.provider :libvirt do |libvirt| From c208582e75c53dbbf22349379b5bcb0cb13dc35d Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sat, 30 Oct 2021 19:14:12 +0200 Subject: [PATCH 10/15] Drop old java_paths --- buildserver/config.buildserver.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/buildserver/config.buildserver.yml b/buildserver/config.buildserver.yml index f5fff843..944535c5 100644 --- a/buildserver/config.buildserver.yml +++ b/buildserver/config.buildserver.yml @@ -1,6 +1,2 @@ sdk_path: /opt/android-sdk - -java_paths: - 8: /usr/lib/jvm/java-8-openjdk-amd64 - gradle_version_dir: /opt/gradle/versions From 046d3c8dcf5a0f58cf3eec9cbc3ff489863753ed Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sat, 30 Oct 2021 20:58:42 +0200 Subject: [PATCH 11/15] Use ssh_config.d --- buildserver/setup-env-vars | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/buildserver/setup-env-vars b/buildserver/setup-env-vars index a5f53fd8..d7d8d245 100644 --- a/buildserver/setup-env-vars +++ b/buildserver/setup-env-vars @@ -18,5 +18,9 @@ echo 'export fdroidserver=$home_vagrant/fdroidserver' >> $bsenv chmod 0644 $bsenv # make sure that SSH never hangs at a password or key prompt -printf ' StrictHostKeyChecking yes\n' >> /etc/ssh/ssh_config -printf ' BatchMode yes\n' >> /etc/ssh/ssh_config +mkdir -p /etc/ssh/ssh_config.d/ +cat << EOF >> /etc/ssh/ssh_config.d/fdroid +Host * + StrictHostKeyChecking yes + BatchMode yes +EOF From c6f59565378b9f1aed83f08666f439a6f1a3ca45 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 18 Apr 2022 10:17:12 +0200 Subject: [PATCH 12/15] install Vagrant insecure_private_key into buildserver box Vagrant uses the "insecure private key" to establish the first SSH connection to a new VM based on a clean public box. In theory, the `vagrant package` command should do that automatically. This process is still using custom code instead of `vagrant package`, hence this script. This public key can be generated on any Vagrant install using: ssh-keygen -y -f ~/.vagrant.d/insecure_private_key https://www.vagrantup.com/docs/vagrantfile/ssh_settings#config-ssh-private_key_path fdroid-bootstrap-buildserver#12 !1012 !1099 --- buildserver/Vagrantfile | 2 ++ .../provision-vagrant-insecure-private-key | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100755 buildserver/provision-vagrant-insecure-private-key diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 5a420b62..57509ffe 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -77,6 +77,8 @@ Vagrant.configure("2") do |config| owner: 'root', group: 'root', create: true end + config.vm.provision "shell", name: "vagrant-insecure-private-key", + path: "provision-vagrant-insecure-private-key" config.vm.provision "shell", name: "setup-env-vars", path: "setup-env-vars", args: ["/opt/android-sdk"] config.vm.provision "shell", name: "apt-get-install", path: "provision-apt-get-install", diff --git a/buildserver/provision-vagrant-insecure-private-key b/buildserver/provision-vagrant-insecure-private-key new file mode 100755 index 00000000..b6ed681e --- /dev/null +++ b/buildserver/provision-vagrant-insecure-private-key @@ -0,0 +1,17 @@ +#!/bin/bash -e +# +# Vagrant uses the "insecure private key" to establish the first SSH +# connection to a new VM based on a clean public box. In theory, the +# `vagrant package` command should do that automatically. This +# process is still using custom code instead of `vagrant package`, +# hence this script. +# +# https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/issues/12 +# https://www.vagrantup.com/docs/vagrantfile/ssh_settings#config-ssh-private_key_path +# +# This public key can be generated using: +# ssh-keygen -y -f ~/.vagrant.d/insecure_private_key + +echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== ~/.vagrant.d/insecure_private_key" > /home/vagrant/.ssh/authorized_keys +chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys +chmod 600 /home/vagrant/.ssh/authorized_keys From 4eb189ad2e8677bbc1e3a01d1e91cfc3d0df4527 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 24 Aug 2022 21:23:42 +0200 Subject: [PATCH 13/15] gitlab-ci: plugin_fetchsrclibs runs in same base OS as fdroiddata --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7bfe690d..5e9a47f1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -407,7 +407,7 @@ fdroid build: # fdroiddata because that one is known to work, and this is a CI job, # so it should be isolated from the normal churn of fdroiddata. plugin_fetchsrclibs: - image: debian:buster + image: debian:bullseye <<: *apt-template only: changes: @@ -430,7 +430,8 @@ plugin_fetchsrclibs: - . env/bin/activate - export PATH="$CI_PROJECT_DIR:$PATH" - export PYTHONPATH="$CI_PROJECT_DIR/examples" - - $pip install wheel # to make this go away: "error: invalid command 'bdist_wheel'" + # workaround https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252 + - export SETUPTOOLS_USE_DISTUTILS=stdlib - $pip install -e . - fdroid | grep fetchsrclibs From a0ea27632e61d06980364a4e7d19b68967c6b957 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 5 Sep 2022 18:45:17 +0200 Subject: [PATCH 14/15] buildserver: use HTTPS for security.debian.org It is now officially supported: https://guardianproject.info/2021/12/08/debian-over-https/ --- buildserver/provision-apt-get-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install index b4855f51..a975ac74 100644 --- a/buildserver/provision-apt-get-install +++ b/buildserver/provision-apt-get-install @@ -38,7 +38,7 @@ fi cat << EOF > /etc/apt/sources.list deb ${debian_mirror} bullseye main -deb http://security.debian.org/debian-security bullseye-security main +deb https://security.debian.org/debian-security bullseye-security main deb ${debian_mirror} bullseye-updates main EOF echo "deb ${debian_mirror} bullseye-backports main" > /etc/apt/sources.list.d/backports.list From 4e839ac63b15de835d63dc153fc94987a0cf907f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 10 Oct 2022 21:22:32 +0200 Subject: [PATCH 15/15] switch to latest vagrant boxes --- makebuildserver | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/makebuildserver b/makebuildserver index 68d07547..aec0192e 100755 --- a/makebuildserver +++ b/makebuildserver @@ -44,8 +44,21 @@ logging.basicConfig(format=logformat, level=loglevel) tail = None BASEBOX_DEFAULT = 'fdroid/bullseye64' -BASEBOX_VERSION_DEFAULT = '11.20220317.1' +BASEBOX_VERSION_DEFAULT = "11.20221010.1" BASEBOX_CHECKSUMS = { + "11.20221010.1": { + "libvirt": { + "box.img": "c2114aa276c176fa65b8072f5dcd1e8a6ab9f7d15fd5da791727a0164fd43254", + "Vagrantfile": "f9c6fcbb47a4d0d33eb066859c8e87efd642287a638bd7da69a9e7a6f25fec47", + "metadata.json": "42b96a01106c25f3a222ddad0baead0b811cc64926f924fb836bbfa43580e646", + }, + "virtualbox": { + "box.ovf": "5e4de5f1f4b481b2c1917c0b2f6e6334f4741cc18c5b278e3bafb094535ff2cb", + "box.vmdk": "737053bc886037ae76bb38a1776eba2a5579d49423de990e93ef4a3f0cab4f1c", + "Vagrantfile": "0bbc2ae97668d8da27ab97b766752dcd0bf9e41900e21057de15a58ee7fae47d", + "metadata.json": "ffdaa989f2f6932cd8042e1102371f405cc7ad38e324210a1326192e4689e83a", + } + }, '11.20220317.1': { 'libvirt': { 'box.img': 'fbde152a2f61d191983be9d1dbeae2591af32cca1ec27daa342485d97187515e',