1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-11 15:13:27 +02:00

Merge branch 'master' into 'master'

make it really easy to upgrade unsigned repos to signed

As a key step to removing support for unsigned repos from fdroidclient (https://gitlab.com/fdroid/fdroidclient/issues/12), this merge request makes `fdroid update` require a signing key.  If there is no keystore, it'll prompt the user to create one using `fdroid update --create-key`.

This closes #13

See merge request !48
This commit is contained in:
Ciaran Gultnieks 2015-05-14 16:09:40 +00:00
commit afa7254a83
5 changed files with 290 additions and 111 deletions

View File

@ -30,6 +30,9 @@ import Queue
import threading
import magic
import logging
import hashlib
import socket
from distutils.version import LooseVersion
from zipfile import ZipFile
@ -61,7 +64,7 @@ default_config = {
'stats_to_carbon': False,
'repo_maxage': 0,
'build_server_always': False,
'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
'Summary': 50,
@ -2016,3 +2019,63 @@ def find_command(command):
return exe_file
return None
def genpassword():
'''generate a random password for when generating keys'''
h = hashlib.sha256()
h.update(os.urandom(16)) # salt
h.update(bytes(socket.getfqdn()))
return h.digest().encode('base64').strip()
def genkeystore(localconfig):
'''Generate a new key with random passwords and add it to new keystore'''
logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
keystoredir = os.path.dirname(localconfig['keystore'])
if keystoredir is None or keystoredir == '':
keystoredir = os.path.join(os.getcwd(), keystoredir)
if not os.path.exists(keystoredir):
os.makedirs(keystoredir, mode=0o700)
write_password_file("keystorepass", localconfig['keystorepass'])
write_password_file("keypass", localconfig['keypass'])
p = FDroidPopen(['keytool', '-genkey',
'-keystore', localconfig['keystore'],
'-alias', localconfig['repo_keyalias'],
'-keyalg', 'RSA', '-keysize', '4096',
'-sigalg', 'SHA256withRSA',
'-validity', '10000',
'-storepass:file', config['keystorepassfile'],
'-keypass:file', config['keypassfile'],
'-dname', localconfig['keydname']])
# TODO keypass should be sent via stdin
os.chmod(localconfig['keystore'], 0o0600)
if p.returncode != 0:
raise BuildException("Failed to generate key", p.output)
# now show the lovely key that was just generated
p = FDroidPopen(['keytool', '-list', '-v',
'-keystore', localconfig['keystore'],
'-alias', localconfig['repo_keyalias'],
'-storepass:file', config['keystorepassfile']])
logging.info(p.output.strip() + '\n\n')
def write_to_config(thisconfig, key, value=None):
'''write a key/value to the local config.py'''
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
with open('config.py', 'r') as f:
data = f.read()
pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
repl = '\n' + key + ' = "' + value + '"'
data = re.sub(pattern, repl, data)
# if this key is not in the file, append it
if not re.match('\s*' + key + '\s*=\s*"', data):
data += repl
# make sure the file ends with a carraige return
if not re.match('\n$', data):
data += '\n'
with open('config.py', 'w') as f:
f.writelines(data)

View File

@ -20,7 +20,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import glob
import hashlib
import os
import re
import shutil
@ -30,26 +29,11 @@ from optparse import OptionParser
import logging
import common
from common import FDroidPopen, BuildException
config = {}
options = None
def write_to_config(thisconfig, key, value=None):
'''write a key/value to the local config.py'''
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
with open('config.py', 'r') as f:
data = f.read()
pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
repl = '\n' + key + ' = "' + value + '"'
data = re.sub(pattern, repl, data)
with open('config.py', 'w') as f:
f.writelines(data)
def disable_in_config(key, value):
'''write a key/value to the local config.py, then comment it out'''
with open('config.py', 'r') as f:
@ -61,37 +45,6 @@ def disable_in_config(key, value):
f.writelines(data)
def genpassword():
'''generate a random password for when generating keys'''
h = hashlib.sha256()
h.update(os.urandom(16)) # salt
h.update(bytes(socket.getfqdn()))
return h.digest().encode('base64').strip()
def genkey(keystore, repo_keyalias, password, keydname):
'''generate a new keystore with a new key in it for signing repos'''
logging.info('Generating a new key in "' + keystore + '"...')
common.write_password_file("keystorepass", password)
common.write_password_file("keypass", password)
p = FDroidPopen(['keytool', '-genkey',
'-keystore', keystore, '-alias', repo_keyalias,
'-keyalg', 'RSA', '-keysize', '4096',
'-sigalg', 'SHA256withRSA',
'-validity', '10000',
'-storepass:file', config['keystorepassfile'],
'-keypass:file', config['keypassfile'],
'-dname', keydname])
# TODO keypass should be sent via stdin
if p.returncode != 0:
raise BuildException("Failed to generate key", p.output)
# now show the lovely key that was just generated
p = FDroidPopen(['keytool', '-list', '-v',
'-keystore', keystore, '-alias', repo_keyalias,
'-storepass:file', config['keystorepassfile']])
logging.info(p.output.strip() + '\n\n')
def main():
global options, config
@ -171,7 +124,7 @@ def main():
# If android_home is not None, the path given from the command line
# will be directly written in the config.
if 'sdk_path' in test_config:
write_to_config(test_config, 'sdk_path', options.android_home)
common.write_to_config(test_config, 'sdk_path', options.android_home)
else:
logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
logging.info('Try running `fdroid init` in an empty directory.')
@ -197,7 +150,7 @@ def main():
test_config['build_tools'] = ''
else:
test_config['build_tools'] = dirname
write_to_config(test_config, 'build_tools')
common.write_to_config(test_config, 'build_tools')
common.ensure_build_tools_exists(test_config)
# now that we have a local config.py, read configuration...
@ -222,21 +175,21 @@ def main():
if not os.path.exists(keystore):
logging.info('"' + keystore
+ '" does not exist, creating a new keystore there.')
write_to_config(test_config, 'keystore', keystore)
common.write_to_config(test_config, 'keystore', keystore)
repo_keyalias = None
if options.repo_keyalias:
repo_keyalias = options.repo_keyalias
write_to_config(test_config, 'repo_keyalias', repo_keyalias)
common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
if options.distinguished_name:
keydname = options.distinguished_name
write_to_config(test_config, 'keydname', keydname)
common.write_to_config(test_config, 'keydname', keydname)
if keystore == 'NONE': # we're using a smartcard
write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default
common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default
disable_in_config('keypass', 'never used with smartcard')
write_to_config(test_config, 'smartcardoptions',
('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
+ '-providerClass sun.security.pkcs11.SunPKCS11 '
+ '-providerArg opensc-fdroid.cfg'))
common.write_to_config(test_config, 'smartcardoptions',
('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
+ '-providerClass sun.security.pkcs11.SunPKCS11 '
+ '-providerArg opensc-fdroid.cfg'))
# find opensc-pkcs11.so
if not os.path.exists('opensc-fdroid.cfg'):
if os.path.exists('/usr/lib/opensc-pkcs11.so'):
@ -258,20 +211,17 @@ def main():
with open('opensc-fdroid.cfg', 'w') as f:
f.write(opensc_fdroid)
elif not os.path.exists(keystore):
# no existing or specified keystore, generate the whole thing
keystoredir = os.path.dirname(keystore)
if not os.path.exists(keystoredir):
os.makedirs(keystoredir, mode=0o700)
password = genpassword()
write_to_config(test_config, 'keystorepass', password)
write_to_config(test_config, 'keypass', password)
if options.repo_keyalias is None:
repo_keyalias = socket.getfqdn()
write_to_config(test_config, 'repo_keyalias', repo_keyalias)
if not options.distinguished_name:
keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
write_to_config(test_config, 'keydname', keydname)
genkey(keystore, repo_keyalias, password, keydname)
password = common.genpassword()
c = dict(test_config)
c['keystorepass'] = password
c['keypass'] = password
c['repo_keyalias'] = socket.getfqdn()
c['keydname'] = 'CN=' + c['repo_keyalias'] + ', OU=F-Droid'
common.write_to_config(test_config, 'keystorepass', password)
common.write_to_config(test_config, 'keypass', password)
common.write_to_config(test_config, 'repo_keyalias', c['repo_keyalias'])
common.write_to_config(test_config, 'keydname', c['keydname'])
common.genkeystore(c)
logging.info('Built repo based in "' + fdroiddir + '"')
logging.info('with this config:')

View File

@ -23,6 +23,7 @@ import os
import shutil
import glob
import re
import socket
import zipfile
import hashlib
import pickle
@ -664,6 +665,36 @@ def scan_apks(apps, apkcache, repodir, knownapks):
repo_pubkey_fingerprint = None
# Generate a certificate fingerprint the same way keytool does it
# (but with slightly different formatting)
def cert_fingerprint(data):
digest = hashlib.sha256(data).digest()
ret = []
ret.append(' '.join("%02X" % ord(b) for b in digest))
return " ".join(ret)
def extract_pubkey():
global repo_pubkey_fingerprint
if 'repo_pubkey' in config:
pubkey = unhexlify(config['repo_pubkey'])
else:
p = FDroidPopen(['keytool', '-exportcert',
'-alias', config['repo_keyalias'],
'-keystore', config['keystore'],
'-storepass:file', config['keystorepassfile']]
+ config['smartcardoptions'], output=False)
if p.returncode != 0 or len(p.output) < 20:
msg = "Failed to get repo pubkey!"
if config['keystore'] == 'NONE':
msg += ' Is your crypto smartcard plugged in?'
logging.critical(msg)
sys.exit(1)
pubkey = p.output
repo_pubkey_fingerprint = cert_fingerprint(pubkey)
return hexlify(pubkey)
def make_index(apps, sortedids, apks, repodir, archive, categories):
"""Make a repo index.
@ -711,38 +742,28 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
repoel.setAttribute("version", "12")
repoel.setAttribute("timestamp", str(int(time.time())))
if 'repo_keyalias' in config:
# Generate a certificate fingerprint the same way keytool does it
# (but with slightly different formatting)
def cert_fingerprint(data):
digest = hashlib.sha256(data).digest()
ret = []
ret.append(' '.join("%02X" % ord(b) for b in digest))
return " ".join(ret)
def extract_pubkey():
global repo_pubkey_fingerprint
if 'repo_pubkey' in config:
pubkey = unhexlify(config['repo_pubkey'])
else:
p = FDroidPopen(['keytool', '-exportcert',
'-alias', config['repo_keyalias'],
'-keystore', config['keystore'],
'-storepass:file', config['keystorepassfile']]
+ config['smartcardoptions'], output=False)
if p.returncode != 0:
msg = "Failed to get repo pubkey!"
if config['keystore'] == 'NONE':
msg += ' Is your crypto smartcard plugged in?'
logging.critical(msg)
sys.exit(1)
pubkey = p.output
repo_pubkey_fingerprint = cert_fingerprint(pubkey)
return hexlify(pubkey)
repoel.setAttribute("pubkey", extract_pubkey())
nosigningkey = False
if not 'repo_keyalias' in config:
nosigningkey = True
logging.critical("'repo_keyalias' not found in config.py!")
if not 'keystore' in config:
nosigningkey = True
logging.critical("'keystore' not found in config.py!")
if not 'keystorepass' in config:
nosigningkey = True
logging.critical("'keystorepass' not found in config.py!")
if not 'keypass' in config:
nosigningkey = True
logging.critical("'keypass' not found in config.py!")
if not os.path.exists(config['keystore']):
nosigningkey = True
logging.critical("'" + config['keystore'] + "' does not exist!")
if nosigningkey:
logging.warning("`fdroid update` requires a signing key, you can create one using:")
logging.warning("\tfdroid update --create-key")
sys.exit(1)
repoel.setAttribute("pubkey", extract_pubkey())
root.appendChild(repoel)
for appid in sortedids:
@ -995,6 +1016,8 @@ def main():
# Parse command line...
parser = OptionParser()
parser.add_option("--create-key", action="store_true", default=False,
help="Create a repo signing key in a keystore")
parser.add_option("-c", "--create-metadata", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("--delete-unknown", action="store_true", default=False,
@ -1041,6 +1064,32 @@ def main():
logging.critical(k + ' "' + config[k] + '" does not exist! Correct it in config.py.')
sys.exit(1)
# if the user asks to create a keystore, do it now, reusing whatever it can
if options.create_key:
if os.path.exists(config['keystore']):
logging.critical("Cowardily refusing to overwrite existing signing key setup!")
logging.critical("\t'" + config['keystore'] + "'")
sys.exit(1)
if not 'repo_keyalias' in config:
config['repo_keyalias'] = socket.getfqdn()
common.write_to_config(config, 'repo_keyalias', config['repo_keyalias'])
if not 'keydname' in config:
config['keydname'] = 'CN=' + config['repo_keyalias'] + ', OU=F-Droid'
common.write_to_config(config, 'keydname', config['keydname'])
if not 'keystore' in config:
config['keystore'] = common.default_config.keystore
common.write_to_config(config, 'keystore', config['keystore'])
password = common.genpassword()
if not 'keystorepass' in config:
config['keystorepass'] = password
common.write_to_config(config, 'keystorepass', config['keystorepass'])
if not 'keypass' in config:
config['keypass'] = password
common.write_to_config(config, 'keypass', config['keypass'])
common.genkeystore(config)
# Get all apps...
apps = metadata.read_metadata()

View File

@ -41,8 +41,12 @@ export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
#------------------------------------------------------------------------------#
# run local tests, don't scan fdroidserver/ project for APKs
# this is a local repo on the Guardian Project Jenkins server
apksource=/var/www/fdroid
cd $WORKSPACE/tests
./run-tests ~jenkins/workspace/[[:upper:]a-eg-z]\*
./run-tests $apksource
#------------------------------------------------------------------------------#
@ -62,7 +66,7 @@ python2 setup.py install
# run tests in new pip+virtualenv install
. $WORKSPACE/env/bin/activate
fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/
fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource
#------------------------------------------------------------------------------#

View File

@ -160,7 +160,7 @@ cd $REPOROOT
$fdroid init
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
grep -F '<application id=' repo/index.xml
grep -F '<application id=' repo/index.xml > /dev/null
LOCALCOPYDIR=`create_test_dir`/fdroid
$fdroid server update --local-copy-dir=$LOCALCOPYDIR
@ -263,7 +263,7 @@ $fdroid init --keystore $KEYSTORE --android-home $STORED_ANDROID_HOME --no-promp
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
grep -F '<application id=' repo/index.xml
grep -F '<application id=' repo/index.xml > /dev/null
test -e repo/index.xml
test -e repo/index.jar
export ANDROID_HOME=$STORED_ANDROID_HOME
@ -278,7 +278,7 @@ mkdir repo
copy_apks_into_repo $REPOROOT
$fdroid init
$fdroid update --create-metadata
grep -F '<application id=' repo/index.xml
grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
@ -293,7 +293,34 @@ copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
grep -F '<application id=' repo/index.xml
grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
echo_header "setup a new repo manually and generate a keystore"
REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
touch config.py
cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/
! test -e $KEYSTORE
set +e
$fdroid update
if [ $? -eq 0 ]; then
echo "This should have failed because this repo has no keystore!"
exit 1
else
echo "`fdroid update` prompted to add keystore"
fi
set -e
$fdroid update --create-key
test -e $KEYSTORE
copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
@ -308,12 +335,12 @@ copy_apks_into_repo $REPOROOT
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
grep -F '<application id=' repo/index.xml
grep -F '<application id=' repo/index.xml > /dev/null
cp $WORKSPACE/tests/urzip.apk $REPOROOT/
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
grep -F '<application id=' repo/index.xml
grep -F '<application id=' repo/index.xml > /dev/null
#------------------------------------------------------------------------------#
@ -325,6 +352,92 @@ $fdroid init --keystore NONE
test -e opensc-fdroid.cfg
test ! -e NONE
#------------------------------------------------------------------------------#
echo_header "setup a new repo with no keystore, add APK, and update"
REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
touch config.py
touch fdroid-icon.png
mkdir repo/
cp $WORKSPACE/tests/urzip.apk $REPOROOT/
set +e
$fdroid update --create-metadata
if [ $? -eq 0 ]; then
echo "This should have failed because this repo has no keystore!"
exit 1
else
echo "`fdroid update` prompted to add keystore"
fi
set -e
# now set up fake, non-working keystore setup
touch $KEYSTORE
echo "keystore = \"$KEYSTORE\"" >> config.py
echo 'repo_keyalias = "foo"' >> config.py
echo 'keystorepass = "foo"' >> config.py
echo 'keypass = "foo"' >> config.py
set +e
$fdroid update --create-metadata
if [ $? -eq 0 ]; then
echo "This should have failed because this repo has a bad/fake keystore!"
exit 1
else
echo "`fdroid update` prompted to add keystore"
fi
set -e
#------------------------------------------------------------------------------#
echo_header "setup a new repo with keystore with APK, update, then without key"
REPOROOT=`create_test_dir`
KEYSTORE=$REPOROOT/keystore.jks
cd $REPOROOT
$fdroid init --keystore $KEYSTORE
test -e $KEYSTORE
cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
$fdroid update --create-metadata
test -e repo/index.xml
test -e repo/index.jar
grep -F '<application id=' repo/index.xml > /dev/null
# now set fake repo_keyalias
sed -i 's,^ *repo_keyalias.*,repo_keyalias = "fake",' $REPOROOT/config.py
set +e
$fdroid update
if [ $? -eq 0 ]; then
echo "This should have failed because this repo has a bad repo_keyalias!"
exit 1
else
echo "`fdroid update` prompted to add keystore"
fi
set -e
# try creating a new keystore, but fail because the old one is there
test -e $KEYSTORE
set +e
$fdroid update --create-key
if [ $? -eq 0 ]; then
echo "This should have failed because a keystore is already there!"
exit 1
else
echo "`fdroid update` complained about existing keystore"
fi
set -e
# now actually create the key with the existing settings
rm -f $KEYSTORE
! test -e $KEYSTORE
$fdroid update --create-key
test -e $KEYSTORE
#------------------------------------------------------------------------------#
# remove this to prevent git conflicts and complaining
rm -rf $WORKSPACE/fdroidserver.egg-info/
echo SUCCESS