diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 96b9b59b..e20f7ddd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -41,7 +41,7 @@ metadata_v0:
   image: registry.gitlab.com/fdroid/fdroidserver:buildserver
   variables:
     GIT_DEPTH: 1000
-    RELEASE_COMMIT_ID: b987a4af5c770472f7db8958b304b28b1e6624fd  # 2.1.1
+    RELEASE_COMMIT_ID: 58cfce106b6d68dc8ebde7842cf01225f5adfd1b  # 2.2b
   script:
     - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
     - cd tests
@@ -56,16 +56,6 @@ metadata_v0:
     - git checkout $GITCOMMIT
     - cd fdroiddata
     - ../tests/dump_internal_metadata_format.py
-    - sed -i
-          -e "s/CurrentVersionCode:.'\([0-9]*\)'/CurrentVersionCode:\1/"
-          -e "s/  versionCode:.'\([0-9]*\)'/  versionCode:\1/"
-          -e "s/  timeout:.'\([0-9]*\)'/  timeout:\1/"
-          -e "/VercodeOperation/s/null/[]/"
-          -e 's/VercodeOperation:.\([^[]\+\)/VercodeOperation:\n- \1/'
-          -e '/LiberapayID/d'
-          -e '/postbuild/d'
-          -e '/binary:/d'
-          metadata/dump_*/*.yaml
     - diff -uw metadata/dump_*
 
 .apt-template: &apt-template
diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 65b19db4..3d5224ff 100644
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -3349,6 +3349,8 @@ def sign_apk(unsigned_path, signed_path, keyalias):
         'FDROID_KEY_STORE_PASS': config['keystorepass'],
         'FDROID_KEY_PASS': config.get('keypass', "")})
     if p.returncode != 0:
+        if os.path.exists(signed_path):
+            os.remove(signed_path)
         raise BuildException(_("Failed to sign application"), p.output)
     os.remove(unsigned_path)
 
diff --git a/tests/common.TestCase b/tests/common.TestCase
index d2e9adb9..485c0b70 100755
--- a/tests/common.TestCase
+++ b/tests/common.TestCase
@@ -856,6 +856,52 @@ class CommonTest(unittest.TestCase):
         self.assertFalse(os.path.isfile(unsigned))
         self.assertTrue(fdroidserver.common.verify_apk_signature(signed))
 
+    @unittest.skipIf(os.getuid() == 0, 'This is meaningless when run as root')
+    def test_sign_apk_fail(self):
+        config = fdroidserver.common.read_config(fdroidserver.common.options)
+        if 'apksigner' not in config:
+            self.skipTest('SKIPPING test_sign_apk_fail, apksigner not installed!')
+
+        config['keyalias'] = 'sova'
+        config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        config['keystore'] = os.path.join(self.basedir, 'keystore.jks')
+        fdroidserver.common.config = config
+        fdroidserver.signindex.config = config
+
+        unsigned = os.path.join(self.testdir, 'urzip-release-unsigned.apk')
+        signed = os.path.join(self.testdir, 'urzip-release.apk')
+        shutil.copy(os.path.join(self.basedir, 'urzip-release-unsigned.apk'), self.testdir)
+
+        os.chmod(unsigned, 0o000)
+        with self.assertRaises(fdroidserver.exception.BuildException):
+            fdroidserver.common.sign_apk(unsigned, signed, config['keyalias'])
+        os.chmod(unsigned, 0o777)
+        self.assertTrue(os.path.isfile(unsigned))
+        self.assertFalse(os.path.isfile(signed))
+
+    def test_sign_apk_corrupt(self):
+        config = fdroidserver.common.read_config(fdroidserver.common.options)
+        if 'apksigner' not in config:
+            self.skipTest('SKIPPING test_sign_apk_corrupt, apksigner not installed!')
+
+        config['keyalias'] = 'sova'
+        config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        config['keystore'] = os.path.join(self.basedir, 'keystore.jks')
+        fdroidserver.common.config = config
+        fdroidserver.signindex.config = config
+
+        unsigned = os.path.join(self.testdir, 'urzip-release-unsigned.apk')
+        signed = os.path.join(self.testdir, 'urzip-release.apk')
+        with open(unsigned, 'w') as fp:
+            fp.write('this is a corrupt APK')
+
+        with self.assertRaises(fdroidserver.exception.BuildException):
+            fdroidserver.common.sign_apk(unsigned, signed, config['keyalias'])
+        self.assertTrue(os.path.isfile(unsigned))
+        self.assertFalse(os.path.isfile(signed))
+
     @unittest.skipUnless(
         os.path.exists('tests/SystemWebView-repack.apk'), "file too big for sdist"
     )