Merge branch 'index-parsing' into 'master'

Download and return repository index

See merge request !240

fdroidserver/common.py

@@ -42,10 +42,17 @@ from distutils.version import LooseVersion
 from queue import Queue
 from zipfile import ZipFile
 
+from pyasn1.codec.der import decoder, encoder
+from pyasn1_modules import rfc2315
+from pyasn1.error import PyAsn1Error
+
 import fdroidserver.metadata
 from .asynchronousfilereader import AsynchronousFileReader
 
 
+# A signature block file with a .DSA, .RSA, or .EC extension
+CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
+
 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
 
 config = None
@@ -2027,16 +2034,21 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir):
     return None
 
 
-def verify_apk_signature(apk):
+def verify_apk_signature(apk, jar=False):
     """verify the signature on an APK
 
     Try to use apksigner whenever possible since jarsigner is very
     shitty: unsigned APKs pass as "verified"! So this has to turn on
     -strict then check for result 4.
 
+    You can set :param: jar to True if you want to use this method
+    to verify jar signatures.
     """
     if set_command_in_config('apksigner'):
-        return subprocess.call([config['apksigner'], 'verify', apk]) == 0
+        args = [config['apksigner'], 'verify']
+        if jar:
+            args += ['--min-sdk-version=1']
+        return subprocess.call(args + [apk]) == 0
     else:
         logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
         return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
@@ -2213,6 +2225,26 @@ def get_cert_fingerprint(pubkey):
     return " ".join(ret)
 
 
+def get_certificate(certificate_file):
+    """
+    Extracts a certificate from the given file.
+    :param certificate_file: file bytes (as string) representing the certificate
+    :return: A binary representation of the certificate's public key, or None in case of error
+    """
+    content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
+    if content.getComponentByName('contentType') != rfc2315.signedData:
+        return None
+    content = decoder.decode(content.getComponentByName('content'),
+                             asn1Spec=rfc2315.SignedData())[0]
+    try:
+        certificates = content.getComponentByName('certificates')
+        cert = certificates[0].getComponentByName('certificate')
+    except PyAsn1Error:
+        logging.error("Certificates not found.")
+        return None
+    return encoder.encode(cert)
+
+
 def write_to_config(thisconfig, key, value=None, config_file=None):
     '''write a key/value to the local config.py
 

fdroidserver/index.py

@@ -28,11 +28,15 @@ import os
 import re
 import shutil
 import sys
+import tempfile
 import urllib.parse
+import zipfile
 from binascii import hexlify, unhexlify
 from datetime import datetime
 from xml.dom.minidom import Document
 
+import requests
+
 from fdroidserver import metadata, signindex, common
 from fdroidserver.common import FDroidPopen, FDroidPopenBytes
 from fdroidserver.metadata import MetaDataException
@@ -535,3 +539,87 @@ def get_raw_mirror(url):
 
     url = "/".join(url)
     return url
+
+
+class VerificationException(Exception):
+    pass
+
+
+def download_repo_index(url_str, verify_fingerprint=True):
+    """
+    Downloads the repository index from the given :param url_str
+    and verifies the repository's fingerprint if :param verify_fingerprint is not False.
+
+    :raises: VerificationException() if the repository could not be verified
+
+    :return: The index in JSON format.
+    """
+    url = urllib.parse.urlsplit(url_str)
+
+    fingerprint = None
+    if verify_fingerprint:
+        query = urllib.parse.parse_qs(url.query)
+        if 'fingerprint' not in query:
+            raise VerificationException("No fingerprint in URL.")
+        fingerprint = query['fingerprint'][0]
+
+    url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
+    r = requests.get(url.geturl())
+
+    with tempfile.NamedTemporaryFile() as fp:
+        # write and open JAR file
+        fp.write(r.content)
+        jar = zipfile.ZipFile(fp)
+
+        # verify that the JAR signature is valid
+        verify_jar_signature(fp.name)
+
+        # get public key and its fingerprint from JAR
+        public_key, public_key_fingerprint = get_public_key_from_jar(jar)
+
+        # compare the fingerprint if verify_fingerprint is True
+        if verify_fingerprint and fingerprint.upper() != public_key_fingerprint:
+            raise VerificationException("The repository's fingerprint does not match.")
+
+        # load repository index from JSON
+        index = json.loads(jar.read('index-v1.json').decode("utf-8"))
+        index["repo"]["pubkey"] = hexlify(public_key).decode("utf-8")
+        index["repo"]["fingerprint"] = public_key_fingerprint
+
+        # turn the apps into App objects
+        index["apps"] = [metadata.App(app) for app in index["apps"]]
+
+        return index
+
+
+def verify_jar_signature(file):
+    """
+    Verifies the signature of a given JAR file.
+
+    :raises: VerificationException() if the JAR's signature could not be verified
+    """
+    if not common.verify_apk_signature(file, jar=True):
+        raise VerificationException("The repository's index could not be verified.")
+
+
+def get_public_key_from_jar(jar):
+    """
+    Get the public key and its fingerprint from a JAR file.
+
+    :raises: VerificationException() if the JAR was not signed exactly once
+
+    :param jar: a zipfile.ZipFile object
+    :return: the public key from the jar and its fingerprint
+    """
+    # extract certificate from jar
+    certs = [n for n in jar.namelist() if common.CERT_PATH_REGEX.match(n)]
+    if len(certs) < 1:
+        raise VerificationException("Found no signing certificates for repository.")
+    if len(certs) > 1:
+        raise VerificationException("Found multiple signing certificates for repository.")
+
+    # extract public key from certificate
+    public_key = common.get_certificate(jar.read(certs[0]))
+    public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
+
+    return public_key, public_key_fingerprint

fdroidserver/update.py

@@ -34,9 +34,6 @@ from datetime import datetime, timedelta
 from argparse import ArgumentParser
 
 import collections
-from pyasn1.error import PyAsn1Error
-from pyasn1.codec.der import decoder, encoder
-from pyasn1_modules import rfc2315
 from binascii import hexlify
 
 from PIL import Image
@@ -45,7 +42,7 @@ import logging
 from . import common
 from . import index
 from . import metadata
-from .common import FDroidPopen, SdkToolsPopen
+from .common import SdkToolsPopen
 
 METADATA_VERSION = 18
 
@@ -379,10 +376,6 @@ def resize_all_icons(repodirs):
                 resize_icon(iconpath, density)
 
 
-# A signature block file with a .DSA, .RSA, or .EC extension
-cert_path_regex = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
-
-
 def getsig(apkpath):
     """ Get the signing certificate of an apk. To get the same md5 has that
     Android gets, we encode the .RSA certificate in a specific format and pass
@@ -393,18 +386,12 @@ def getsig(apkpath):
               if an error occurred.
     """
 
-    cert = None
-
     # verify the jar signature is correct
-    args = [config['jarsigner'], '-verify', apkpath]
-    p = FDroidPopen(args)
-    if p.returncode != 0:
-        logging.critical(apkpath + " has a bad signature!")
+    if not common.verify_apk_signature(apkpath):
         return None
 
     with zipfile.ZipFile(apkpath, 'r') as apk:
-
-        certs = [n for n in apk.namelist() if cert_path_regex.match(n)]
+        certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)]
 
         if len(certs) < 1:
             logging.error("Found no signing certificates on %s" % apkpath)
@@ -415,20 +402,7 @@ def getsig(apkpath):
 
         cert = apk.read(certs[0])
 
-    content = decoder.decode(cert, asn1Spec=rfc2315.ContentInfo())[0]
-    if content.getComponentByName('contentType') != rfc2315.signedData:
-        logging.error("Unexpected format.")
-        return None
-
-    content = decoder.decode(content.getComponentByName('content'),
-                             asn1Spec=rfc2315.SignedData())[0]
-    try:
-        certificates = content.getComponentByName('certificates')
-    except PyAsn1Error:
-        logging.error("Certificates not found.")
-        return None
-
-    cert_encoded = encoder.encode(certificates)[4:]
+    cert_encoded = common.get_certificate(cert)
 
     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 

tests/index.TestCase

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+import optparse
+import os
+import unittest
+import zipfile
+
+import fdroidserver.common
+import fdroidserver.index
+import fdroidserver.signindex
+
+
+class IndexTest(unittest.TestCase):
+
+    def setUp(self):
+        fdroidserver.common.config = None
+        config = fdroidserver.common.read_config(fdroidserver.common.options)
+        config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
+        fdroidserver.common.config = config
+        fdroidserver.signindex.config = config
+
+    @staticmethod
+    def test_verify_jar_signature_succeeds():
+        basedir = os.path.dirname(__file__)
+        source_dir = os.path.join(basedir, 'signindex')
+        for f in ('testy.jar', 'guardianproject.jar'):
+            testfile = os.path.join(source_dir, f)
+            fdroidserver.index.verify_jar_signature(testfile)
+
+    def test_verify_jar_signature_fails(self):
+        basedir = os.path.dirname(__file__)
+        source_dir = os.path.join(basedir, 'signindex')
+        testfile = os.path.join(source_dir, 'unsigned.jar')
+        with self.assertRaises(fdroidserver.index.VerificationException):
+            fdroidserver.index.verify_jar_signature(testfile)
+
+    def test_get_public_key_from_jar_succeeds(self):
+        basedir = os.path.dirname(__file__)
+        source_dir = os.path.join(basedir, 'signindex')
+        for f in ('testy.jar', 'guardianproject.jar'):
+            testfile = os.path.join(source_dir, f)
+            jar = zipfile.ZipFile(testfile)
+            _, fingerprint = fdroidserver.index.get_public_key_from_jar(jar)
+            # comparing fingerprints should be sufficient
+            if f == 'testy.jar':
+                self.assertTrue(fingerprint ==
+                                '818E469465F96B704E27BE2FEE4C63AB' +
+                                '9F83DDF30E7A34C7371A4728D83B0BC1')
+            if f == 'guardianproject.jar':
+                self.assertTrue(fingerprint ==
+                                'B7C2EEFD8DAC7806AF67DFCD92EB1812' +
+                                '6BC08312A7F2D6F3862E46013C7A6135')
+
+    def test_get_public_key_from_jar_fails(self):
+        basedir = os.path.dirname(__file__)
+        source_dir = os.path.join(basedir, 'signindex')
+        testfile = os.path.join(source_dir, 'unsigned.jar')
+        jar = zipfile.ZipFile(testfile)
+        with self.assertRaises(fdroidserver.index.VerificationException):
+            fdroidserver.index.get_public_key_from_jar(jar)
+
+    def test_download_repo_index_no_fingerprint(self):
+        with self.assertRaises(fdroidserver.index.VerificationException):
+            fdroidserver.index.download_repo_index("http://example.org")
+
+    def test_download_repo_index_no_jar(self):
+        with self.assertRaises(zipfile.BadZipFile):
+            fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope")
+
+    # TODO test_download_repo_index with an actual repository
+
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(IndexTest))
+    unittest.main()