Merge branch 'master' into 'master'

make it really easy to upgrade unsigned repos to signed

As a key step to removing support for unsigned repos from fdroidclient (https://gitlab.com/fdroid/fdroidclient/issues/12), this merge request makes `fdroid update` require a signing key.  If there is no keystore, it'll prompt the user to create one using `fdroid update --create-key`.

This closes #13

See merge request !48

fdroidserver/common.py

@@ -30,6 +30,9 @@ import Queue
 import threading
 import magic
 import logging
+import hashlib
+import socket
+
 from distutils.version import LooseVersion
 from zipfile import ZipFile
 
@@ -61,7 +64,7 @@ default_config = {
     'stats_to_carbon': False,
     'repo_maxage': 0,
     'build_server_always': False,
-    'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
+    'keystore': 'keystore.jks',
     'smartcardoptions': [],
     'char_limits': {
         'Summary': 50,
@@ -2016,3 +2019,63 @@ def find_command(command):
                 return exe_file
 
     return None
+
+
+def genpassword():
+    '''generate a random password for when generating keys'''
+    h = hashlib.sha256()
+    h.update(os.urandom(16))  # salt
+    h.update(bytes(socket.getfqdn()))
+    return h.digest().encode('base64').strip()
+
+
+def genkeystore(localconfig):
+    '''Generate a new key with random passwords and add it to new keystore'''
+    logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
+    keystoredir = os.path.dirname(localconfig['keystore'])
+    if keystoredir is None or keystoredir == '':
+        keystoredir = os.path.join(os.getcwd(), keystoredir)
+    if not os.path.exists(keystoredir):
+        os.makedirs(keystoredir, mode=0o700)
+
+    write_password_file("keystorepass", localconfig['keystorepass'])
+    write_password_file("keypass", localconfig['keypass'])
+    p = FDroidPopen(['keytool', '-genkey',
+                     '-keystore', localconfig['keystore'],
+                     '-alias', localconfig['repo_keyalias'],
+                     '-keyalg', 'RSA', '-keysize', '4096',
+                     '-sigalg', 'SHA256withRSA',
+                     '-validity', '10000',
+                     '-storepass:file', config['keystorepassfile'],
+                     '-keypass:file', config['keypassfile'],
+                     '-dname', localconfig['keydname']])
+    # TODO keypass should be sent via stdin
+    os.chmod(localconfig['keystore'], 0o0600)
+    if p.returncode != 0:
+        raise BuildException("Failed to generate key", p.output)
+    # now show the lovely key that was just generated
+    p = FDroidPopen(['keytool', '-list', '-v',
+                     '-keystore', localconfig['keystore'],
+                     '-alias', localconfig['repo_keyalias'],
+                     '-storepass:file', config['keystorepassfile']])
+    logging.info(p.output.strip() + '\n\n')
+
+
+def write_to_config(thisconfig, key, value=None):
+    '''write a key/value to the local config.py'''
+    if value is None:
+        origkey = key + '_orig'
+        value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
+    with open('config.py', 'r') as f:
+        data = f.read()
+    pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
+    repl = '\n' + key + ' = "' + value + '"'
+    data = re.sub(pattern, repl, data)
+    # if this key is not in the file, append it
+    if not re.match('\s*' + key + '\s*=\s*"', data):
+        data += repl
+    # make sure the file ends with a carraige return
+    if not re.match('\n$', data):
+        data += '\n'
+    with open('config.py', 'w') as f:
+        f.writelines(data)

fdroidserver/init.py

@@ -20,7 +20,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import glob
-import hashlib
 import os
 import re
 import shutil
@@ -30,26 +29,11 @@ from optparse import OptionParser
 import logging
 
 import common
-from common import FDroidPopen, BuildException
 
 config = {}
 options = None
 
 
-def write_to_config(thisconfig, key, value=None):
-    '''write a key/value to the local config.py'''
-    if value is None:
-        origkey = key + '_orig'
-        value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
-    with open('config.py', 'r') as f:
-        data = f.read()
-    pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
-    repl = '\n' + key + ' = "' + value + '"'
-    data = re.sub(pattern, repl, data)
-    with open('config.py', 'w') as f:
-        f.writelines(data)
-
-
 def disable_in_config(key, value):
     '''write a key/value to the local config.py, then comment it out'''
     with open('config.py', 'r') as f:
@@ -61,37 +45,6 @@ def disable_in_config(key, value):
         f.writelines(data)
 
 
-def genpassword():
-    '''generate a random password for when generating keys'''
-    h = hashlib.sha256()
-    h.update(os.urandom(16))  # salt
-    h.update(bytes(socket.getfqdn()))
-    return h.digest().encode('base64').strip()
-
-
-def genkey(keystore, repo_keyalias, password, keydname):
-    '''generate a new keystore with a new key in it for signing repos'''
-    logging.info('Generating a new key in "' + keystore + '"...')
-    common.write_password_file("keystorepass", password)
-    common.write_password_file("keypass", password)
-    p = FDroidPopen(['keytool', '-genkey',
-                     '-keystore', keystore, '-alias', repo_keyalias,
-                     '-keyalg', 'RSA', '-keysize', '4096',
-                     '-sigalg', 'SHA256withRSA',
-                     '-validity', '10000',
-                     '-storepass:file', config['keystorepassfile'],
-                     '-keypass:file', config['keypassfile'],
-                     '-dname', keydname])
-    # TODO keypass should be sent via stdin
-    if p.returncode != 0:
-        raise BuildException("Failed to generate key", p.output)
-    # now show the lovely key that was just generated
-    p = FDroidPopen(['keytool', '-list', '-v',
-                     '-keystore', keystore, '-alias', repo_keyalias,
-                     '-storepass:file', config['keystorepassfile']])
-    logging.info(p.output.strip() + '\n\n')
-
-
 def main():
 
     global options, config
@@ -171,7 +124,7 @@ def main():
         # If android_home is not None, the path given from the command line
         # will be directly written in the config.
         if 'sdk_path' in test_config:
-            write_to_config(test_config, 'sdk_path', options.android_home)
+            common.write_to_config(test_config, 'sdk_path', options.android_home)
     else:
         logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
         logging.info('Try running `fdroid init` in an empty directory.')
@@ -197,7 +150,7 @@ def main():
                 test_config['build_tools'] = ''
             else:
                 test_config['build_tools'] = dirname
-            write_to_config(test_config, 'build_tools')
+            common.write_to_config(test_config, 'build_tools')
         common.ensure_build_tools_exists(test_config)
 
     # now that we have a local config.py, read configuration...
@@ -222,21 +175,21 @@ def main():
             if not os.path.exists(keystore):
                 logging.info('"' + keystore
                              + '" does not exist, creating a new keystore there.')
-    write_to_config(test_config, 'keystore', keystore)
+    common.write_to_config(test_config, 'keystore', keystore)
     repo_keyalias = None
     if options.repo_keyalias:
         repo_keyalias = options.repo_keyalias
-        write_to_config(test_config, 'repo_keyalias', repo_keyalias)
+        common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
     if options.distinguished_name:
         keydname = options.distinguished_name
-        write_to_config(test_config, 'keydname', keydname)
+        common.write_to_config(test_config, 'keydname', keydname)
     if keystore == 'NONE':  # we're using a smartcard
-        write_to_config(test_config, 'repo_keyalias', '1')  # seems to be the default
+        common.write_to_config(test_config, 'repo_keyalias', '1')  # seems to be the default
         disable_in_config('keypass', 'never used with smartcard')
-        write_to_config(test_config, 'smartcardoptions',
+        common.write_to_config(test_config, 'smartcardoptions',
-                        ('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
+                               ('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
-                         + '-providerClass sun.security.pkcs11.SunPKCS11 '
+                                + '-providerClass sun.security.pkcs11.SunPKCS11 '
-                         + '-providerArg opensc-fdroid.cfg'))
+                                + '-providerArg opensc-fdroid.cfg'))
         # find opensc-pkcs11.so
         if not os.path.exists('opensc-fdroid.cfg'):
             if os.path.exists('/usr/lib/opensc-pkcs11.so'):
@@ -258,20 +211,17 @@ def main():
             with open('opensc-fdroid.cfg', 'w') as f:
                 f.write(opensc_fdroid)
     elif not os.path.exists(keystore):
-        # no existing or specified keystore, generate the whole thing
+        password = common.genpassword()
-        keystoredir = os.path.dirname(keystore)
+        c = dict(test_config)
-        if not os.path.exists(keystoredir):
+        c['keystorepass'] = password
-            os.makedirs(keystoredir, mode=0o700)
+        c['keypass'] = password
-        password = genpassword()
+        c['repo_keyalias'] = socket.getfqdn()
-        write_to_config(test_config, 'keystorepass', password)
+        c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid'
-        write_to_config(test_config, 'keypass', password)
+        common.write_to_config(test_config, 'keystorepass', password)
-        if options.repo_keyalias is None:
+        common.write_to_config(test_config, 'keypass', password)
-            repo_keyalias = socket.getfqdn()
+        common.write_to_config(test_config, 'repo_keyalias', c['repo_keyalias'])
-            write_to_config(test_config, 'repo_keyalias', repo_keyalias)
+        common.write_to_config(test_config, 'keydname', c['keydname'])
-        if not options.distinguished_name:
+        common.genkeystore(c)
-            keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
-            write_to_config(test_config, 'keydname', keydname)
-        genkey(keystore, repo_keyalias, password, keydname)
 
     logging.info('Built repo based in "' + fdroiddir + '"')
     logging.info('with this config:')

fdroidserver/update.py

@@ -23,6 +23,7 @@ import os
 import shutil
 import glob
 import re
+import socket
 import zipfile
 import hashlib
 import pickle
@@ -664,6 +665,36 @@ def scan_apks(apps, apkcache, repodir, knownapks):
 repo_pubkey_fingerprint = None
 
 
+# Generate a certificate fingerprint the same way keytool does it
+# (but with slightly different formatting)
+def cert_fingerprint(data):
+    digest = hashlib.sha256(data).digest()
+    ret = []
+    ret.append(' '.join("%02X" % ord(b) for b in digest))
+    return " ".join(ret)
+
+
+def extract_pubkey():
+    global repo_pubkey_fingerprint
+    if 'repo_pubkey' in config:
+        pubkey = unhexlify(config['repo_pubkey'])
+    else:
+        p = FDroidPopen(['keytool', '-exportcert',
+                         '-alias', config['repo_keyalias'],
+                         '-keystore', config['keystore'],
+                         '-storepass:file', config['keystorepassfile']]
+                        + config['smartcardoptions'], output=False)
+        if p.returncode != 0 or len(p.output) < 20:
+            msg = "Failed to get repo pubkey!"
+            if config['keystore'] == 'NONE':
+                msg += ' Is your crypto smartcard plugged in?'
+            logging.critical(msg)
+            sys.exit(1)
+        pubkey = p.output
+    repo_pubkey_fingerprint = cert_fingerprint(pubkey)
+    return hexlify(pubkey)
+
+
 def make_index(apps, sortedids, apks, repodir, archive, categories):
     """Make a repo index.
 
@@ -711,38 +742,28 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
     repoel.setAttribute("version", "12")
     repoel.setAttribute("timestamp", str(int(time.time())))
 
-    if 'repo_keyalias' in config:
+    nosigningkey = False
-
+    if not 'repo_keyalias' in config:
-        # Generate a certificate fingerprint the same way keytool does it
+        nosigningkey = True
-        # (but with slightly different formatting)
+        logging.critical("'repo_keyalias' not found in config.py!")
-        def cert_fingerprint(data):
+    if not 'keystore' in config:
-            digest = hashlib.sha256(data).digest()
+        nosigningkey = True
-            ret = []
+        logging.critical("'keystore' not found in config.py!")
-            ret.append(' '.join("%02X" % ord(b) for b in digest))
+    if not 'keystorepass' in config:
-            return " ".join(ret)
+        nosigningkey = True
-
+        logging.critical("'keystorepass' not found in config.py!")
-        def extract_pubkey():
+    if not 'keypass' in config:
-            global repo_pubkey_fingerprint
+        nosigningkey = True
-            if 'repo_pubkey' in config:
+        logging.critical("'keypass' not found in config.py!")
-                pubkey = unhexlify(config['repo_pubkey'])
+    if not os.path.exists(config['keystore']):
-            else:
+        nosigningkey = True
-                p = FDroidPopen(['keytool', '-exportcert',
+        logging.critical("'" + config['keystore'] + "' does not exist!")
-                                 '-alias', config['repo_keyalias'],
+    if nosigningkey:
-                                 '-keystore', config['keystore'],
+        logging.warning("`fdroid update` requires a signing key, you can create one using:")
-                                 '-storepass:file', config['keystorepassfile']]
+        logging.warning("\tfdroid update --create-key")
-                                + config['smartcardoptions'], output=False)
+        sys.exit(1)
-                if p.returncode != 0:
-                    msg = "Failed to get repo pubkey!"
-                    if config['keystore'] == 'NONE':
-                        msg += ' Is your crypto smartcard plugged in?'
-                    logging.critical(msg)
-                    sys.exit(1)
-                pubkey = p.output
-            repo_pubkey_fingerprint = cert_fingerprint(pubkey)
-            return hexlify(pubkey)
-
-        repoel.setAttribute("pubkey", extract_pubkey())
 
+    repoel.setAttribute("pubkey", extract_pubkey())
     root.appendChild(repoel)
 
     for appid in sortedids:
@@ -995,6 +1016,8 @@ def main():
 
     # Parse command line...
     parser = OptionParser()
+    parser.add_option("--create-key", action="store_true", default=False,
+                      help="Create a repo signing key in a keystore")
     parser.add_option("-c", "--create-metadata", action="store_true", default=False,
                       help="Create skeleton metadata files that are missing")
     parser.add_option("--delete-unknown", action="store_true", default=False,
@@ -1041,6 +1064,32 @@ def main():
                 logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
                 sys.exit(1)
 
+    # if the user asks to create a keystore, do it now, reusing whatever it can
+    if options.create_key:
+        if os.path.exists(config['keystore']):
+            logging.critical("Cowardily refusing to overwrite existing signing key setup!")
+            logging.critical("\t'" + config['keystore'] + "'")
+            sys.exit(1)
+
+        if not 'repo_keyalias' in config:
+            config['repo_keyalias'] = socket.getfqdn()
+            common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
+        if not 'keydname' in config:
+            config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
+            common.write_to_config(config, 'keydname', config['keydname'])
+        if not 'keystore' in config:
+            config['keystore'] = common.default_config.keystore
+            common.write_to_config(config, 'keystore', config['keystore'])
+
+        password = common.genpassword()
+        if not 'keystorepass' in config:
+            config['keystorepass'] = password
+            common.write_to_config(config, 'keystorepass', config['keystorepass'])
+        if not 'keypass' in config:
+            config['keypass'] = password
+            common.write_to_config(config, 'keypass', config['keypass'])
+        common.genkeystore(config)
+
     # Get all apps...
     apps = metadata.read_metadata()
 

jenkins-build

@@ -41,8 +41,12 @@ export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
 
 #------------------------------------------------------------------------------#
 # run local tests, don't scan fdroidserver/ project for APKs
+
+# this is a local repo on the Guardian Project Jenkins server
+apksource=/var/www/fdroid
+
 cd $WORKSPACE/tests
-./run-tests ~jenkins/workspace/[[:upper:]a-eg-z]\*
+./run-tests $apksource
 
 
 #------------------------------------------------------------------------------#
@@ -62,7 +66,7 @@ python2 setup.py install
 
 # run tests in new pip+virtualenv install
 . $WORKSPACE/env/bin/activate
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/
+fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
 
 
 #------------------------------------------------------------------------------#

tests/run-tests

@@ -160,7 +160,7 @@ cd $REPOROOT
 $fdroid init
 copy_apks_into_repo $REPOROOT
 $fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
 
 LOCALCOPYDIR=`create_test_dir`/fdroid
 $fdroid server update --local-copy-dir=$LOCALCOPYDIR
@@ -263,7 +263,7 @@ $fdroid init --keystore $KEYSTORE --android-home $STORED_ANDROID_HOME --no-promp
 test -e $KEYSTORE
 copy_apks_into_repo $REPOROOT
 $fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
 test -e repo/index.xml
 test -e repo/index.jar
 export ANDROID_HOME=$STORED_ANDROID_HOME
@@ -278,7 +278,7 @@ mkdir repo
 copy_apks_into_repo $REPOROOT
 $fdroid init
 $fdroid update --create-metadata
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
 
 
 #------------------------------------------------------------------------------#
@@ -293,7 +293,34 @@ copy_apks_into_repo $REPOROOT
 $fdroid update --create-metadata
 test -e repo/index.xml
 test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo manually and generate a keystore"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+touch config.py
+cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/
+! test -e $KEYSTORE
+set +e
+$fdroid update
+if [ $? -eq 0 ]; then
+    echo "This should have failed because this repo has no keystore!"
+    exit 1
+else
+    echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+$fdroid update --create-key
+test -e $KEYSTORE
+copy_apks_into_repo $REPOROOT
+$fdroid update --create-metadata
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
 
 
 #------------------------------------------------------------------------------#
@@ -308,12 +335,12 @@ copy_apks_into_repo $REPOROOT
 $fdroid update --create-metadata
 test -e repo/index.xml
 test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
 cp $WORKSPACE/tests/urzip.apk $REPOROOT/
 $fdroid update --create-metadata
 test -e repo/index.xml
 test -e repo/index.jar
-grep -F '<application id=' repo/index.xml
+grep -F '<application id=' repo/index.xml > /dev/null
 
 
 #------------------------------------------------------------------------------#
@@ -325,6 +352,92 @@ $fdroid init --keystore NONE
 test -e opensc-fdroid.cfg
 test ! -e NONE
 
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo with no keystore, add APK, and update"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+touch config.py
+touch fdroid-icon.png
+mkdir repo/
+cp $WORKSPACE/tests/urzip.apk $REPOROOT/
+set +e
+$fdroid update --create-metadata
+if [ $? -eq 0 ]; then
+    echo "This should have failed because this repo has no keystore!"
+    exit 1
+else
+    echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+
+# now set up fake, non-working keystore setup
+touch $KEYSTORE
+echo "keystore = \"$KEYSTORE\"" >> config.py
+echo 'repo_keyalias = "foo"' >> config.py
+echo 'keystorepass = "foo"' >> config.py
+echo 'keypass = "foo"' >> config.py
+set +e
+$fdroid update --create-metadata
+if [ $? -eq 0 ]; then
+    echo "This should have failed because this repo has a bad/fake keystore!"
+    exit 1
+else
+    echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo with keystore with APK, update, then without key"
+
+REPOROOT=`create_test_dir`
+KEYSTORE=$REPOROOT/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE
+test -e $KEYSTORE
+cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
+$fdroid update --create-metadata
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
+
+# now set fake repo_keyalias
+sed -i 's,^ *repo_keyalias.*,repo_keyalias = "fake",' $REPOROOT/config.py
+set +e
+$fdroid update
+if [ $? -eq 0 ]; then
+    echo "This should have failed because this repo has a bad repo_keyalias!"
+    exit 1
+else
+    echo "`fdroid update` prompted to add keystore"
+fi
+set -e
+
+# try creating a new keystore, but fail because the old one is there
+test -e $KEYSTORE
+set +e
+$fdroid update --create-key
+if [ $? -eq 0 ]; then
+    echo "This should have failed because a keystore is already there!"
+    exit 1
+else
+    echo "`fdroid update` complained about existing keystore"
+fi
+set -e
+
+# now actually create the key with the existing settings
+rm -f $KEYSTORE
+! test -e $KEYSTORE
+$fdroid update --create-key
+test -e $KEYSTORE
+
+
+#------------------------------------------------------------------------------#
+
+# remove this to prevent git conflicts and complaining
 rm -rf $WORKSPACE/fdroidserver.egg-info/
 
 echo SUCCESS