convert apkcache from pickle to JSON

pickle can serialize executable code, while JSON is only ever pure data.
The APK cache is only ever pure data, so no need for the security risks of
pickle.  For example, if some malicious thing gets write access on the
`fdroid update` machine, it can write out a custom tmp/apkcache which would
then be executed.  That is not possible with JSON.

This does just ignore any existing cache and rebuilds from scratch. That is
so we don't need to maintain pickle anywhere, and to ensure there are no
glitches from a conversion from pickle to JSON.

closes #163

fdroidserver/index.py

@@ -533,7 +533,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
                 old_permissions = set()
                 sorted_permissions = sorted(apk['uses-permission'])
                 for perm in sorted_permissions:
-                    perm_name = perm.name
+                    perm_name = perm[0]
                     if perm_name.startswith("android.permission."):
                         perm_name = perm_name[19:]
                     old_permissions.add(perm_name)
@@ -541,15 +541,15 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
 
                 for permission in sorted_permissions:
                     permel = doc.createElement('uses-permission')
-                    permel.setAttribute('name', permission.name)
+                    permel.setAttribute('name', permission[0])
-                    if permission.maxSdkVersion is not None:
+                    if permission[1] is not None:
-                        permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
+                        permel.setAttribute('maxSdkVersion', '%d' % permission[1])
                         apkel.appendChild(permel)
                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
                     permel = doc.createElement('uses-permission-sdk-23')
-                    permel.setAttribute('name', permission_sdk_23.name)
+                    permel.setAttribute('name', permission_sdk_23[0])
-                    if permission_sdk_23.maxSdkVersion is not None:
+                    if permission_sdk_23[1] is not None:
-                        permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
+                        permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23[1])
                         apkel.appendChild(permel)
                 if 'nativecode' in apk:
                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)

fdroidserver/update.py

@@ -27,7 +27,7 @@ import re
 import socket
 import zipfile
 import hashlib
-import pickle  # nosec TODO
+import json
 import time
 import copy
 from datetime import datetime
@@ -46,7 +46,7 @@ from . import metadata
 from .common import SdkToolsPopen
 from .exception import BuildException, FDroidException
 
-METADATA_VERSION = 19
+METADATA_VERSION = 20
 
 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
 UNSET_VERSION_CODE = -0x100000000
@@ -56,8 +56,7 @@ APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
 APK_LABEL_ICON_PAT = re.compile(r".*\s+label='(.*)'\s+icon='(.*?)'")
 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
-APK_PERMISSION_PAT = \
+APK_PERMISSION_PAT = re.compile(".*(name='(.*)')(.*maxSdkVersion='(.*)')?.*")
-    re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
 
 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
@@ -438,7 +437,7 @@ def getsig(apkpath):
 
 
 def get_cache_file():
-    return os.path.join('tmp', 'apkcache')
+    return os.path.join('tmp', 'apkcache.json')
 
 
 def get_cache():
@@ -460,27 +459,46 @@ def get_cache():
     apkcachefile = get_cache_file()
     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
     if not options.clean and os.path.exists(apkcachefile):
-        with open(apkcachefile, 'rb') as cf:
+        with open(apkcachefile) as fp:
-            apkcache = pickle.load(cf, encoding='utf-8')  # nosec TODO
+            apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict)
         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
            or apkcache.get('allow_disabled_algorithms') != ada:
-            apkcache = {}
+            apkcache = collections.OrderedDict()
     else:
-        apkcache = {}
+        apkcache = collections.OrderedDict()
 
     apkcache["METADATA_VERSION"] = METADATA_VERSION
     apkcache['allow_disabled_algorithms'] = ada
 
+    for k, v in apkcache.items():
+        if not isinstance(v, dict):
+            continue
+        if 'antiFeatures' in v:
+            v['antiFeatures'] = set(v['antiFeatures'])
+        if 'added' in v:
+            v['added'] = datetime.fromtimestamp(v['added'])
+
     return apkcache
 
 
 def write_cache(apkcache):
+    class Encoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, set):
+                return ['SET'] + list(obj)
+            elif isinstance(obj, datetime):
+                return obj.timestamp()
+            return super().default(obj)
+
     apkcachefile = get_cache_file()
     cache_path = os.path.dirname(apkcachefile)
     if not os.path.exists(cache_path):
         os.makedirs(cache_path)
-    with open(apkcachefile, 'wb') as cf:
+    for k, v in apkcache.items():
-        pickle.dump(apkcache, cf)
+        if isinstance(k, bytes):
+            print('BYTES: ' + str(k) + ' ' + str(v))
+    with open(apkcachefile, 'w') as fp:
+        json.dump(apkcache, fp, cls=Encoder, indent=2)
 
 
 def get_icon_bytes(apkzip, iconsrc):
@@ -948,16 +966,16 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 
     cachechanged = False
     repo_files = []
-    repodir = repodir.encode('utf-8')
+    repodir = repodir.encode()
     for name in os.listdir(repodir):
         file_extension = common.get_file_extension(name)
         if file_extension == 'apk' or file_extension == 'obb':
             continue
         filename = os.path.join(repodir, name)
-        name_utf8 = name.decode('utf-8')
+        name_utf8 = name.decode()
         if filename.endswith(b'_src.tar.gz'):
             logging.debug(_('skipping source tarball: {path}')
-                          .format(path=filename.decode('utf-8')))
+                          .format(path=filename.decode()))
             continue
         if not common.is_repo_file(filename):
             continue
@@ -968,15 +986,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 
         shasum = sha256sum(filename)
         usecache = False
-        if name in apkcache:
+        if name_utf8 in apkcache:
-            repo_file = apkcache[name]
+            repo_file = apkcache[name_utf8]
-            # added time is cached as tuple but used here as datetime instance
-            if 'added' in repo_file:
-                a = repo_file['added']
-                if isinstance(a, datetime):
-                    repo_file['added'] = a
-                else:
-                    repo_file['added'] = datetime(*a[:6])
             if repo_file.get('hash') == shasum:
                 logging.debug(_("Reading {apkfilename} from cache")
                               .format(apkfilename=name_utf8))
@@ -1004,10 +1015,10 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
                 repo_file['versionCode'] = int(m.group(2))
             srcfilename = name + b'_src.tar.gz'
             if os.path.exists(os.path.join(repodir, srcfilename)):
-                repo_file['srcname'] = srcfilename.decode('utf-8')
+                repo_file['srcname'] = srcfilename.decode()
             repo_file['size'] = stat.st_size
 
-            apkcache[name] = repo_file
+            apkcache[name_utf8] = repo_file
             cachechanged = True
 
         if use_date_from_file:

tests/repo/index-v1.json

@@ -1,7 +1,7 @@
 {
   "repo": {
     "timestamp": 1502845383782,
-    "version": 19,
+    "version": 20,
     "name": "My First F-Droid Repo Demo",
     "icon": "fdroid-icon.png",
     "address": "https://MyFirstFDroidRepo.org/fdroid/repo",

tests/repo/index.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <fdroid>
-	<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="19">
+	<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="20">
 		<description>This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/u/fdroid. </description>
 		<mirror>http://foobarfoobarfoobar.onion/fdroid/repo</mirror>
 		<mirror>https://foo.bar/fdroid/repo</mirror>

tests/run-tests

@@ -177,8 +177,8 @@ if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then
     test -e repo/index.xml
     test -e repo/index.jar
     test -e repo/index-v1.jar
-    test -e tmp/apkcache
+    test -e tmp/apkcache.json
-    ! test -z tmp/apkcache
+    ! test -z tmp/apkcache.json
     test -L urzip.apk
     grep -F '<application id=' repo/index.xml > /dev/null
 fi
@@ -808,8 +808,8 @@ else
     test -e repo/index.xml
     test -e repo/index.jar
     test -e repo/index-v1.jar
-    test -e tmp/apkcache
+    test -e tmp/apkcache.json
-    ! test -z tmp/apkcache
+    ! test -z tmp/apkcache.json
     export ANDROID_HOME=$STORED_ANDROID_HOME
 fi
 
@@ -860,8 +860,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -890,8 +890,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -917,8 +917,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -1009,8 +1009,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 # now set fake repo_keyalias

tests/update.TestCase

@@ -3,6 +3,7 @@
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
 import git
+import glob
 import inspect
 import logging
 import optparse
@@ -289,6 +290,45 @@ class UpdateTest(unittest.TestCase):
                 self.assertIsNone(apk.get('obbMainFile'))
                 self.assertIsNone(apk.get('obbPatchFile'))
 
+    def test_apkcache_json(self):
+        """test the migration from pickle to json"""
+        os.chdir(os.path.join(localmodule, 'tests'))
+        if os.path.basename(os.getcwd()) != 'tests':
+            raise Exception('This test must be run in the "tests/" subdir')
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        config['ndk_paths'] = dict()
+        config['accepted_formats'] = ['json', 'txt', 'yml']
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+
+        fdroidserver.update.options = type('', (), {})()
+        fdroidserver.update.options.clean = True
+        fdroidserver.update.options.delete_unknown = True
+        fdroidserver.update.options.rename_apks = False
+        fdroidserver.update.options.allow_disabled_algorithms = False
+
+        fdroidserver.metadata.read_metadata(xref=True)
+        knownapks = fdroidserver.common.KnownApks()
+        apkcache = fdroidserver.update.get_cache()
+        self.assertEqual(2, len(apkcache))
+        self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"])
+        self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms,
+                         apkcache['allow_disabled_algorithms'])
+        apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False)
+        fdroidserver.update.write_cache(apkcache)
+
+        fdroidserver.update.options.clean = False
+        read_from_json = fdroidserver.update.get_cache()
+        self.assertEqual(16, len(read_from_json))
+        for f in glob.glob('repo/*.apk'):
+            self.assertTrue(os.path.basename(f) in read_from_json)
+
+        fdroidserver.update.options.clean = True
+        reset = fdroidserver.update.get_cache()
+        self.assertEqual(2, len(reset))
+
     def test_scan_apk(self):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)