Merge branch 'smaller-methods' into 'master'

Factor out code into smaller methods to be used by repomaker

See merge request !236

fdroidserver/common.py

@@ -37,6 +37,7 @@ import base64
 import zipfile
 import xml.etree.ElementTree as XMLElementTree
 
+from binascii import hexlify
 from datetime import datetime
 from distutils.version import LooseVersion
 from queue import Queue
@@ -2197,7 +2198,10 @@ def genpassword():
 
 
 def genkeystore(localconfig):
-    '''Generate a new key with random passwords and add it to new keystore'''
+    """
+    Generate a new key with password provided in :param localconfig and add it to new keystore
+    :return: hexed public key, public key fingerprint
+    """
     logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
     keystoredir = os.path.dirname(localconfig['keystore'])
     if keystoredir is None or keystoredir == '':
@@ -2220,12 +2224,35 @@ def genkeystore(localconfig):
     if p.returncode != 0:
         raise BuildException("Failed to generate key", p.output)
     os.chmod(localconfig['keystore'], 0o0600)
-    # now show the lovely key that was just generated
+    if not options.quiet:
-    p = FDroidPopen([config['keytool'], '-list', '-v',
+        # now show the lovely key that was just generated
-                     '-keystore', localconfig['keystore'],
+        p = FDroidPopen([config['keytool'], '-list', '-v',
-                     '-alias', localconfig['repo_keyalias'],
+                         '-keystore', localconfig['keystore'],
-                     '-storepass:file', config['keystorepassfile']])
+                         '-alias', localconfig['repo_keyalias'],
-    logging.info(p.output.strip() + '\n\n')
+                         '-storepass:file', config['keystorepassfile']])
+        logging.info(p.output.strip() + '\n\n')
+    # get the public key
+    p = FDroidPopenBytes([config['keytool'], '-exportcert',
+                          '-keystore', localconfig['keystore'],
+                          '-alias', localconfig['repo_keyalias'],
+                          '-storepass:file', config['keystorepassfile']]
+                         + config['smartcardoptions'],
+                         output=False, stderr_to_stdout=False)
+    if p.returncode != 0 or len(p.output) < 20:
+        raise BuildException("Failed to get public key", p.output)
+    pubkey = p.output
+    fingerprint = get_cert_fingerprint(pubkey)
+    return hexlify(pubkey), fingerprint
+
+
+def get_cert_fingerprint(pubkey):
+    """
+    Generate a certificate fingerprint the same way keytool does it
+    (but with slightly different formatting)
+    """
+    digest = hashlib.sha256(pubkey).digest()
+    ret = [' '.join("%02X" % b for b in bytearray(digest))]
+    return " ".join(ret)
 
 
 def write_to_config(thisconfig, key, value=None):

fdroidserver/update.py

@@ -52,6 +52,9 @@ from .metadata import MetaDataException
 
 METADATA_VERSION = 18
 
+# less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
+UNSET_VERSION_CODE = -0x100000000
+
 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
@@ -433,6 +436,38 @@ def getsig(apkpath):
     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 
 
+def get_cache_file():
+    return os.path.join('tmp', 'apkcache')
+
+
+def get_cache():
+    """
+    Gather information about all the apk files in the repo directory,
+    using cached data if possible.
+    :return: apkcache
+    """
+    apkcachefile = get_cache_file()
+    if not options.clean and os.path.exists(apkcachefile):
+        with open(apkcachefile, 'rb') as cf:
+            apkcache = pickle.load(cf, encoding='utf-8')
+        if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
+            apkcache = {}
+    else:
+        apkcache = {}
+
+    return apkcache
+
+
+def write_cache(apkcache):
+    apkcachefile = get_cache_file()
+    cache_path = os.path.dirname(apkcachefile)
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
+    apkcache["METADATA_VERSION"] = METADATA_VERSION
+    with open(apkcachefile, 'wb') as cf:
+        pickle.dump(apkcache, cf)
+
+
 def get_icon_bytes(apkzip, iconsrc):
     '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
     try:
@@ -1072,15 +1107,6 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
 repo_pubkey_fingerprint = None
 
 
-# Generate a certificate fingerprint the same way keytool does it
-# (but with slightly different formatting)
-def cert_fingerprint(data):
-    digest = hashlib.sha256(data).digest()
-    ret = []
-    ret.append(' '.join("%02X" % b for b in bytearray(digest)))
-    return " ".join(ret)
-
-
 def extract_pubkey():
     global repo_pubkey_fingerprint
     if 'repo_pubkey' in config:
@@ -1099,10 +1125,49 @@ def extract_pubkey():
             logging.critical(msg)
             sys.exit(1)
         pubkey = p.output
-    repo_pubkey_fingerprint = cert_fingerprint(pubkey)
+    repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey)
     return hexlify(pubkey)
 
 
+def apply_info_from_latest_apk(apps, apks):
+    """
+    Some information from the apks needs to be applied up to the application level.
+    When doing this, we use the info from the most recent version's apk.
+    We deal with figuring out when the app was added and last updated at the same time.
+    """
+    for appid, app in apps.items():
+        bestver = UNSET_VERSION_CODE
+        for apk in apks:
+            if apk['packageName'] == appid:
+                if apk['versionCode'] > bestver:
+                    bestver = apk['versionCode']
+                    bestapk = apk
+
+                if 'added' in apk:
+                    if not app.added or apk['added'] < app.added:
+                        app.added = apk['added']
+                    if not app.lastUpdated or apk['added'] > app.lastUpdated:
+                        app.lastUpdated = apk['added']
+
+        if not app.added:
+            logging.debug("Don't know when " + appid + " was added")
+        if not app.lastUpdated:
+            logging.debug("Don't know when " + appid + " was last updated")
+
+        if bestver == UNSET_VERSION_CODE:
+
+            if app.Name is None:
+                app.Name = app.AutoName or appid
+            app.icon = None
+            logging.debug("Application " + appid + " has no packages")
+        else:
+            if app.Name is None:
+                app.Name = bestapk['name']
+            app.icon = bestapk['icon'] if 'icon' in bestapk else None
+            if app.CurrentVersionCode is None:
+                app.CurrentVersionCode = str(bestver)
+
+
 # Get raw URL from git service for mirroring
 def get_raw_mirror(url):
     # Divide urls in parts
@@ -1839,17 +1904,10 @@ def main():
     # Read known apks data (will be updated and written back when we've finished)
     knownapks = common.KnownApks()
 
-    # Gather information about all the apk files in the repo directory, using
+    # Get APK cache
-    # cached data if possible.
+    apkcache = get_cache()
-    apkcachefile = os.path.join('tmp', 'apkcache')
-    if not options.clean and os.path.exists(apkcachefile):
-        with open(apkcachefile, 'rb') as cf:
-            apkcache = pickle.load(cf, encoding='utf-8')
-        if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
-            apkcache = {}
-    else:
-        apkcache = {}
 
+    # Delete builds for disabled apps
     delete_disabled_builds(apps, apkcache, repodirs)
 
     # Scan all apks in the main repo
@@ -1909,44 +1967,8 @@ def main():
     else:
         archapks = []
 
-    # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
+    # Apply information from latest apks to the application and update dates
-    UNSET_VERSION_CODE = -0x100000000
+    apply_info_from_latest_apk(apps, apks + archapks)
-
-    # Some information from the apks needs to be applied up to the application
-    # level. When doing this, we use the info from the most recent version's apk.
-    # We deal with figuring out when the app was added and last updated at the
-    # same time.
-    for appid, app in apps.items():
-        bestver = UNSET_VERSION_CODE
-        for apk in apks + archapks:
-            if apk['packageName'] == appid:
-                if apk['versionCode'] > bestver:
-                    bestver = apk['versionCode']
-                    bestapk = apk
-
-                if 'added' in apk:
-                    if not app.added or apk['added'] < app.added:
-                        app.added = apk['added']
-                    if not app.lastUpdated or apk['added'] > app.lastUpdated:
-                        app.lastUpdated = apk['added']
-
-        if not app.added:
-            logging.debug("Don't know when " + appid + " was added")
-        if not app.lastUpdated:
-            logging.debug("Don't know when " + appid + " was last updated")
-
-        if bestver == UNSET_VERSION_CODE:
-
-            if app.Name is None:
-                app.Name = app.AutoName or appid
-            app.icon = None
-            logging.debug("Application " + appid + " has no packages")
-        else:
-            if app.Name is None:
-                app.Name = bestapk['name']
-            app.icon = bestapk['icon'] if 'icon' in bestapk else None
-            if app.CurrentVersionCode is None:
-                app.CurrentVersionCode = str(bestver)
 
     # Sort the app list by name, then the web site doesn't have to by default.
     # (we had to wait until we'd scanned the apks to do this, because mostly the
@@ -1983,7 +2005,6 @@ def main():
         make_binary_transparency_log(repodirs)
 
     if config['update_stats']:
-
         # Update known apks info...
         knownapks.writeifchanged()
 
@@ -2003,9 +2024,7 @@ def main():
                 f.write(data)
 
     if cachechanged:
-        apkcache["METADATA_VERSION"] = METADATA_VERSION
+        write_cache(apkcache)
-        with open(apkcachefile, 'wb') as cf:
-            pickle.dump(apkcache, cf)
 
     # Update the wiki...
     if options.wiki: