diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 9618727a..985afb4a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -30,6 +30,9 @@ import Queue import threading import magic import logging +import hashlib +import socket + from distutils.version import LooseVersion from zipfile import ZipFile @@ -61,7 +64,7 @@ default_config = { 'stats_to_carbon': False, 'repo_maxage': 0, 'build_server_always': False, - 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'), + 'keystore': 'keystore.jks', 'smartcardoptions': [], 'char_limits': { 'Summary': 50, @@ -2016,3 +2019,63 @@ def find_command(command): return exe_file return None + + +def genpassword(): + '''generate a random password for when generating keys''' + h = hashlib.sha256() + h.update(os.urandom(16)) # salt + h.update(bytes(socket.getfqdn())) + return h.digest().encode('base64').strip() + + +def genkeystore(localconfig): + '''Generate a new key with random passwords and add it to new keystore''' + logging.info('Generating a new key in "' + localconfig['keystore'] + '"...') + keystoredir = os.path.dirname(localconfig['keystore']) + if keystoredir is None or keystoredir == '': + keystoredir = os.path.join(os.getcwd(), keystoredir) + if not os.path.exists(keystoredir): + os.makedirs(keystoredir, mode=0o700) + + write_password_file("keystorepass", localconfig['keystorepass']) + write_password_file("keypass", localconfig['keypass']) + p = FDroidPopen(['keytool', '-genkey', + '-keystore', localconfig['keystore'], + '-alias', localconfig['repo_keyalias'], + '-keyalg', 'RSA', '-keysize', '4096', + '-sigalg', 'SHA256withRSA', + '-validity', '10000', + '-storepass:file', config['keystorepassfile'], + '-keypass:file', config['keypassfile'], + '-dname', localconfig['keydname']]) + # TODO keypass should be sent via stdin + os.chmod(localconfig['keystore'], 0o0600) + if p.returncode != 0: + raise BuildException("Failed to generate key", p.output) + # now show the lovely key that was just generated + p = FDroidPopen(['keytool', '-list', '-v', + '-keystore', localconfig['keystore'], + '-alias', localconfig['repo_keyalias'], + '-storepass:file', config['keystorepassfile']]) + logging.info(p.output.strip() + '\n\n') + + +def write_to_config(thisconfig, key, value=None): + '''write a key/value to the local config.py''' + if value is None: + origkey = key + '_orig' + value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key] + with open('config.py', 'r') as f: + data = f.read() + pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"' + repl = '\n' + key + ' = "' + value + '"' + data = re.sub(pattern, repl, data) + # if this key is not in the file, append it + if not re.match('\s*' + key + '\s*=\s*"', data): + data += repl + # make sure the file ends with a carraige return + if not re.match('\n$', data): + data += '\n' + with open('config.py', 'w') as f: + f.writelines(data) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 2e16efbd..c49cb303 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -20,7 +20,6 @@ # along with this program. If not, see . import glob -import hashlib import os import re import shutil @@ -30,26 +29,11 @@ from optparse import OptionParser import logging import common -from common import FDroidPopen, BuildException config = {} options = None -def write_to_config(thisconfig, key, value=None): - '''write a key/value to the local config.py''' - if value is None: - origkey = key + '_orig' - value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key] - with open('config.py', 'r') as f: - data = f.read() - pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"' - repl = '\n' + key + ' = "' + value + '"' - data = re.sub(pattern, repl, data) - with open('config.py', 'w') as f: - f.writelines(data) - - def disable_in_config(key, value): '''write a key/value to the local config.py, then comment it out''' with open('config.py', 'r') as f: @@ -61,37 +45,6 @@ def disable_in_config(key, value): f.writelines(data) -def genpassword(): - '''generate a random password for when generating keys''' - h = hashlib.sha256() - h.update(os.urandom(16)) # salt - h.update(bytes(socket.getfqdn())) - return h.digest().encode('base64').strip() - - -def genkey(keystore, repo_keyalias, password, keydname): - '''generate a new keystore with a new key in it for signing repos''' - logging.info('Generating a new key in "' + keystore + '"...') - common.write_password_file("keystorepass", password) - common.write_password_file("keypass", password) - p = FDroidPopen(['keytool', '-genkey', - '-keystore', keystore, '-alias', repo_keyalias, - '-keyalg', 'RSA', '-keysize', '4096', - '-sigalg', 'SHA256withRSA', - '-validity', '10000', - '-storepass:file', config['keystorepassfile'], - '-keypass:file', config['keypassfile'], - '-dname', keydname]) - # TODO keypass should be sent via stdin - if p.returncode != 0: - raise BuildException("Failed to generate key", p.output) - # now show the lovely key that was just generated - p = FDroidPopen(['keytool', '-list', '-v', - '-keystore', keystore, '-alias', repo_keyalias, - '-storepass:file', config['keystorepassfile']]) - logging.info(p.output.strip() + '\n\n') - - def main(): global options, config @@ -171,7 +124,7 @@ def main(): # If android_home is not None, the path given from the command line # will be directly written in the config. if 'sdk_path' in test_config: - write_to_config(test_config, 'sdk_path', options.android_home) + common.write_to_config(test_config, 'sdk_path', options.android_home) else: logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') logging.info('Try running `fdroid init` in an empty directory.') @@ -197,7 +150,7 @@ def main(): test_config['build_tools'] = '' else: test_config['build_tools'] = dirname - write_to_config(test_config, 'build_tools') + common.write_to_config(test_config, 'build_tools') common.ensure_build_tools_exists(test_config) # now that we have a local config.py, read configuration... @@ -222,21 +175,21 @@ def main(): if not os.path.exists(keystore): logging.info('"' + keystore + '" does not exist, creating a new keystore there.') - write_to_config(test_config, 'keystore', keystore) + common.write_to_config(test_config, 'keystore', keystore) repo_keyalias = None if options.repo_keyalias: repo_keyalias = options.repo_keyalias - write_to_config(test_config, 'repo_keyalias', repo_keyalias) + common.write_to_config(test_config, 'repo_keyalias', repo_keyalias) if options.distinguished_name: keydname = options.distinguished_name - write_to_config(test_config, 'keydname', keydname) + common.write_to_config(test_config, 'keydname', keydname) if keystore == 'NONE': # we're using a smartcard - write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default + common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default disable_in_config('keypass', 'never used with smartcard') - write_to_config(test_config, 'smartcardoptions', - ('-storetype PKCS11 -providerName SunPKCS11-OpenSC ' - + '-providerClass sun.security.pkcs11.SunPKCS11 ' - + '-providerArg opensc-fdroid.cfg')) + common.write_to_config(test_config, 'smartcardoptions', + ('-storetype PKCS11 -providerName SunPKCS11-OpenSC ' + + '-providerClass sun.security.pkcs11.SunPKCS11 ' + + '-providerArg opensc-fdroid.cfg')) # find opensc-pkcs11.so if not os.path.exists('opensc-fdroid.cfg'): if os.path.exists('/usr/lib/opensc-pkcs11.so'): @@ -258,20 +211,17 @@ def main(): with open('opensc-fdroid.cfg', 'w') as f: f.write(opensc_fdroid) elif not os.path.exists(keystore): - # no existing or specified keystore, generate the whole thing - keystoredir = os.path.dirname(keystore) - if not os.path.exists(keystoredir): - os.makedirs(keystoredir, mode=0o700) - password = genpassword() - write_to_config(test_config, 'keystorepass', password) - write_to_config(test_config, 'keypass', password) - if options.repo_keyalias is None: - repo_keyalias = socket.getfqdn() - write_to_config(test_config, 'repo_keyalias', repo_keyalias) - if not options.distinguished_name: - keydname = 'CN=' + repo_keyalias + ', OU=F-Droid' - write_to_config(test_config, 'keydname', keydname) - genkey(keystore, repo_keyalias, password, keydname) + password = common.genpassword() + c = dict(test_config) + c['keystorepass'] = password + c['keypass'] = password + c['repo_keyalias'] = socket.getfqdn() + c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid' + common.write_to_config(test_config, 'keystorepass', password) + common.write_to_config(test_config, 'keypass', password) + common.write_to_config(test_config, 'repo_keyalias', c['repo_keyalias']) + common.write_to_config(test_config, 'keydname', c['keydname']) + common.genkeystore(c) logging.info('Built repo based in "' + fdroiddir + '"') logging.info('with this config:') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index ee7435e7..48ac7c56 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -23,6 +23,7 @@ import os import shutil import glob import re +import socket import zipfile import hashlib import pickle @@ -664,6 +665,36 @@ def scan_apks(apps, apkcache, repodir, knownapks): repo_pubkey_fingerprint = None +# Generate a certificate fingerprint the same way keytool does it +# (but with slightly different formatting) +def cert_fingerprint(data): + digest = hashlib.sha256(data).digest() + ret = [] + ret.append(' '.join("%02X" % ord(b) for b in digest)) + return " ".join(ret) + + +def extract_pubkey(): + global repo_pubkey_fingerprint + if 'repo_pubkey' in config: + pubkey = unhexlify(config['repo_pubkey']) + else: + p = FDroidPopen(['keytool', '-exportcert', + '-alias', config['repo_keyalias'], + '-keystore', config['keystore'], + '-storepass:file', config['keystorepassfile']] + + config['smartcardoptions'], output=False) + if p.returncode != 0 or len(p.output) < 20: + msg = "Failed to get repo pubkey!" + if config['keystore'] == 'NONE': + msg += ' Is your crypto smartcard plugged in?' + logging.critical(msg) + sys.exit(1) + pubkey = p.output + repo_pubkey_fingerprint = cert_fingerprint(pubkey) + return hexlify(pubkey) + + def make_index(apps, sortedids, apks, repodir, archive, categories): """Make a repo index. @@ -711,38 +742,28 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): repoel.setAttribute("version", "12") repoel.setAttribute("timestamp", str(int(time.time()))) - if 'repo_keyalias' in config: - - # Generate a certificate fingerprint the same way keytool does it - # (but with slightly different formatting) - def cert_fingerprint(data): - digest = hashlib.sha256(data).digest() - ret = [] - ret.append(' '.join("%02X" % ord(b) for b in digest)) - return " ".join(ret) - - def extract_pubkey(): - global repo_pubkey_fingerprint - if 'repo_pubkey' in config: - pubkey = unhexlify(config['repo_pubkey']) - else: - p = FDroidPopen(['keytool', '-exportcert', - '-alias', config['repo_keyalias'], - '-keystore', config['keystore'], - '-storepass:file', config['keystorepassfile']] - + config['smartcardoptions'], output=False) - if p.returncode != 0: - msg = "Failed to get repo pubkey!" - if config['keystore'] == 'NONE': - msg += ' Is your crypto smartcard plugged in?' - logging.critical(msg) - sys.exit(1) - pubkey = p.output - repo_pubkey_fingerprint = cert_fingerprint(pubkey) - return hexlify(pubkey) - - repoel.setAttribute("pubkey", extract_pubkey()) + nosigningkey = False + if not 'repo_keyalias' in config: + nosigningkey = True + logging.critical("'repo_keyalias' not found in config.py!") + if not 'keystore' in config: + nosigningkey = True + logging.critical("'keystore' not found in config.py!") + if not 'keystorepass' in config: + nosigningkey = True + logging.critical("'keystorepass' not found in config.py!") + if not 'keypass' in config: + nosigningkey = True + logging.critical("'keypass' not found in config.py!") + if not os.path.exists(config['keystore']): + nosigningkey = True + logging.critical("'" + config['keystore'] + "' does not exist!") + if nosigningkey: + logging.warning("`fdroid update` requires a signing key, you can create one using:") + logging.warning("\tfdroid update --create-key") + sys.exit(1) + repoel.setAttribute("pubkey", extract_pubkey()) root.appendChild(repoel) for appid in sortedids: @@ -995,6 +1016,8 @@ def main(): # Parse command line... parser = OptionParser() + parser.add_option("--create-key", action="store_true", default=False, + help="Create a repo signing key in a keystore") parser.add_option("-c", "--create-metadata", action="store_true", default=False, help="Create skeleton metadata files that are missing") parser.add_option("--delete-unknown", action="store_true", default=False, @@ -1041,6 +1064,32 @@ def main(): logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.') sys.exit(1) + # if the user asks to create a keystore, do it now, reusing whatever it can + if options.create_key: + if os.path.exists(config['keystore']): + logging.critical("Cowardily refusing to overwrite existing signing key setup!") + logging.critical("\t'" + config['keystore'] + "'") + sys.exit(1) + + if not 'repo_keyalias' in config: + config['repo_keyalias'] = socket.getfqdn() + common.write_to_config(config, 'repo_keyalias', config['repo_keyalias']) + if not 'keydname' in config: + config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid' + common.write_to_config(config, 'keydname', config['keydname']) + if not 'keystore' in config: + config['keystore'] = common.default_config.keystore + common.write_to_config(config, 'keystore', config['keystore']) + + password = common.genpassword() + if not 'keystorepass' in config: + config['keystorepass'] = password + common.write_to_config(config, 'keystorepass', config['keystorepass']) + if not 'keypass' in config: + config['keypass'] = password + common.write_to_config(config, 'keypass', config['keypass']) + common.genkeystore(config) + # Get all apps... apps = metadata.read_metadata() diff --git a/jenkins-build b/jenkins-build index 8a7ca9bd..cb94ebdf 100755 --- a/jenkins-build +++ b/jenkins-build @@ -41,8 +41,12 @@ export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH #------------------------------------------------------------------------------# # run local tests, don't scan fdroidserver/ project for APKs + +# this is a local repo on the Guardian Project Jenkins server +apksource=/var/www/fdroid + cd $WORKSPACE/tests -./run-tests ~jenkins/workspace/[[:upper:]a-eg-z]\* +./run-tests $apksource #------------------------------------------------------------------------------# @@ -62,7 +66,7 @@ python2 setup.py install # run tests in new pip+virtualenv install . $WORKSPACE/env/bin/activate -fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/ +fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource #------------------------------------------------------------------------------# diff --git a/tests/run-tests b/tests/run-tests index d1f988fa..8f14dadc 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -160,7 +160,7 @@ cd $REPOROOT $fdroid init copy_apks_into_repo $REPOROOT $fdroid update --create-metadata -grep -F '> config.py +echo 'keystorepass = "foo"' >> config.py +echo 'keypass = "foo"' >> config.py +set +e +$fdroid update --create-metadata +if [ $? -eq 0 ]; then + echo "This should have failed because this repo has a bad/fake keystore!" + exit 1 +else + echo "`fdroid update` prompted to add keystore" +fi +set -e + + +#------------------------------------------------------------------------------# +echo_header "setup a new repo with keystore with APK, update, then without key" + +REPOROOT=`create_test_dir` +KEYSTORE=$REPOROOT/keystore.jks +cd $REPOROOT +$fdroid init --keystore $KEYSTORE +test -e $KEYSTORE +cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/ +$fdroid update --create-metadata +test -e repo/index.xml +test -e repo/index.jar +grep -F '