diff --git a/jenkins-build-all b/jenkins-build-all
index 2964a9f5..d33f33b4 100755
--- a/jenkins-build-all
+++ b/jenkins-build-all
@@ -74,7 +74,7 @@ if [ -e fdroiddata ]; then
     git checkout master
     git reset --hard origin/master
     # keep all the cloned source repos
-    git clean -fdx --exclude build
+    git clean -fdx --exclude build --exclude repo --exclude unsigned
 else
     git clone https://gitlab.com/fdroid/fdroiddata.git fdroiddata
     cd fdroiddata
diff --git a/jenkins-test b/jenkins-test
index e031d8a4..181da6de 100755
--- a/jenkins-test
+++ b/jenkins-test
@@ -30,11 +30,13 @@ cd tests
 cd $WORKSPACE/fdroiddata
 
 test -e config.py || ../fdroid init --verbose
+test -e keystore.jks || ../fdroid update --create-key
 
 export GNUPGHOME=$WORKSPACE/tests/gnupghome
 echo "build_server_always = True" > config.py
 echo "gpghome = '$GNUPGHOME'" >> config.py
 echo "gpgkey = 'CE71F7FB'" >> config.py
+chmod 0600 config.py
 
 # publish process when building and signing are on separate machines
 test -d repo || mkdir repo