sort index-v1; publish now creates and stores a list of signature fingerprints

fdroidserver/index.py

@@ -39,7 +39,7 @@ from . import common
 from . import metadata
 from . import net
 from . import signindex
-from fdroidserver.common import FDroidPopen, FDroidPopenBytes
+from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
 from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
 
 
@@ -151,11 +151,15 @@ def make(apps, sortedids, apks, repodir, archive):
                 raise TypeError(_('only accepts strings, lists, and tuples'))
         requestsdict[command] = packageNames
 
-    make_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
+    fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
-    make_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
+
+    make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints)
+    make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints)
 
 
-def make_v1(apps, packages, repodir, repodict, requestsdict):
+def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 
     def _index_encoder_default(obj):
         if isinstance(obj, set):
@@ -168,6 +172,9 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
     output['repo'] = repodict
     output['requests'] = requestsdict
 
+    # establish sort order of the index
+    v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints)
+
     appslist = []
     output['apps'] = appslist
     for packageName, appdict in apps.items():
@@ -234,6 +241,35 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
         signindex.sign_index_v1(repodir, json_name)
 
 
+def v1_sort_packages(packages, repodir, fdroid_signing_key_fingerprints):
+
+    GROUP_DEV_SIGNED = 1
+    GROUP_FDROID_SIGNED = 2
+    GROUP_OTHER_SIGNED = 3
+
+    def v1_sort_keys(package):
+        packageName = package.get('packageName', None)
+
+        sig = package.get('signer', None)
+
+        dev_sig = common.metadata_find_developer_signature(packageName)
+        group = GROUP_OTHER_SIGNED
+        if dev_sig and dev_sig == sig:
+            group = GROUP_DEV_SIGNED
+        else:
+            fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
+            if fdroidsig and fdroidsig == sig:
+                group = GROUP_FDROID_SIGNED
+
+        versionCode = None
+        if package.get('versionCode', None):
+            versionCode = -int(package['versionCode'])
+
+        return(packageName, group, sig, versionCode)
+
+    packages.sort(key=v1_sort_keys)
+
+
 def make_v0(apps, apks, repodir, repodict, requestsdict):
     """
     aka index.jar aka index.xml

fdroidserver/publish.py

@@ -53,7 +53,7 @@ def publish_source_tarball(apkfilename, unsigned_dir, output_dir):
 
 
 def key_alias(appid, resolve=False):
-    """Get the alias which which F-Droid uses to indentify the singing key
+    """Get the alias which F-Droid uses to indentify the singing key
     for this App in F-Droids keystore.
     """
     if config and 'keyaliases' in config and appid in config['keyaliases']:
@@ -356,6 +356,9 @@ def main():
                 publish_source_tarball(apkfilename, unsigned_dir, output_dir)
                 logging.info('Published ' + apkfilename)
 
+    store_stats_fdroid_signing_key_fingerprints(allapps.keys())
+    logging.info('published list signing-key fingerprints')
+
 
 if __name__ == "__main__":
     main()

fdroidserver/update.py

@@ -971,13 +971,15 @@ def scan_apk(apk_file):
     else:
         scan_apk_androguard(apk, apk_file)
 
-    # Get the signature
+    # Get the signature, or rather the signing key fingerprints
     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
     apk['sig'] = getsig(apk_file)
     if not apk['sig']:
         raise BuildException("Failed to get apk signature")
     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
                                                                apk_file))
+    if not apk.get('signer'):
+        raise BuildException("Failed to get apk signing key fingerprint")
 
     # Get size of the APK
     apk['size'] = os.path.getsize(apk_file)

tests/index.TestCase

@@ -7,8 +7,10 @@ import sys
 import unittest
 import zipfile
 from unittest.mock import patch
-
 import requests
+import tempfile
+import json
+import shutil
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@@ -19,6 +21,8 @@ if localmodule not in sys.path:
 import fdroidserver.common
 import fdroidserver.index
 import fdroidserver.signindex
+import fdroidserver.publish
+from testcommon import TmpCwd
 
 
 GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
@@ -114,8 +118,117 @@ class IndexTest(unittest.TestCase):
         self.assertEqual(10, len(index['packages']))
         self.assertEqual('new_etag', new_etag)
 
+    def test_v1_sort_packages(self):
+
+        i = [{'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_134.apk',
+              'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
+              'versionCode': 134},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_134_b30bb97.apk',
+              'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
+              'versionCode': 134},
+             {'packageName': 'b075b32b4ef1e8a869e00edb136bd48e34a0382b85ced8628f164d1199584e4e'},
+             {'packageName': '43af70d1aca437c2f9974c4634cc5abe45bdc4d5d71529ac4e553488d3bb3ff6'},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_135_b30bb97.apk',
+              'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
+              'versionCode': 135},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_135.apk',
+              'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
+              'versionCode': 135},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_133.apk',
+              'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
+              'versionCode': 133},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'smssecure-weird-version.apk',
+              'signer': '99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff',
+              'versionCode': 133},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'smssecure-custom.apk',
+              'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
+              'versionCode': 133},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'smssecure-new-custom.apk',
+              'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
+              'versionCode': 135}]
+
+        o = [{'packageName': '43af70d1aca437c2f9974c4634cc5abe45bdc4d5d71529ac4e553488d3bb3ff6'},
+             {'packageName': 'b075b32b4ef1e8a869e00edb136bd48e34a0382b85ced8628f164d1199584e4e'},
+             # app test data
+             # # packages with reproducible developer signature
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_135_b30bb97.apk',
+              'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
+              'versionCode': 135},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_134_b30bb97.apk',
+              'signer': 'b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868',
+              'versionCode': 134},
+             # # packages build and signed by fdroid
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_135.apk',
+              'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
+              'versionCode': 135},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_134.apk',
+              'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
+              'versionCode': 134},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'org.smssecure.smssecure_133.apk',
+              'signer': 'b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6',
+              'versionCode': 133},
+             # # packages signed with unkown keys
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'smssecure-new-custom.apk',
+              'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
+              'versionCode': 135},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'smssecure-custom.apk',
+              'signer': '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef',
+              'versionCode': 133},
+             {'packageName': 'org.smssecure.smssecure',
+              'apkName': 'smssecure-weird-version.apk',
+              'signer': '99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff99ff',
+              'versionCode': 133}]
+
+        fdroidserver.common.config = {}
+        fdroidserver.common.fill_config_defaults(fdroidserver.common.config)
+        fdroidserver.publish.config = fdroidserver.common.config
+        fdroidserver.publish.config['keystorepass'] = '123456'
+        fdroidserver.publish.config['keypass'] = '123456'
+        fdroidserver.publish.config['keystore'] = os.path.join(os.getcwd(),
+                                                               'dummy-keystore.jks')
+        fdroidserver.publish.config['repo_keyalias'] = 'repokey'
+
+        testsmetadir = os.path.join(os.getcwd(), 'metadata')
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            shutil.copytree(testsmetadir, 'metadata')
+            sigkeyfps = {
+                "org.smssecure.smssecure": {
+                    "signer": "b33a601a9da97c82e6eb121eb6b90adab561f396602ec4dc8b0019fb587e2af6"
+                }
+            }
+            os.makedirs('repo')
+            jarfile = 'repo/publishsigkeys.jar'
+            with zipfile.ZipFile(jarfile, 'w', zipfile.ZIP_DEFLATED) as jar:
+                jar.writestr('publishsigkeys.json', json.dumps(sigkeyfps))
+            fdroidserver.publish.sign_sig_key_fingerprint_list(jarfile)
+            with open('config.py', 'w'):
+                pass
+
+            fdroidserver.index.v1_sort_packages(
+                i, 'repo', fdroidserver.common.load_stats_fdroid_signing_key_fingerprints())
+            self.maxDiff = None
+            self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2))
+
 
 if __name__ == "__main__":
+    if os.path.basename(os.getcwd()) != 'tests' and os.path.isdir('tests'):
+        os.chdir('tests')
+
     parser = optparse.OptionParser()
     parser.add_option("-v", "--verbose", action="store_true", default=False,
                       help="Spew out even more information than normal")