Merge branch 'drop-pickle-for-json' into 'master'

Drop pickle for json

Closes #163

See merge request fdroid/fdroidserver!568

fdroidserver/index.py

@@ -533,7 +533,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
                 old_permissions = set()
                 sorted_permissions = sorted(apk['uses-permission'])
                 for perm in sorted_permissions:
-                    perm_name = perm.name
+                    perm_name = perm[0]
                     if perm_name.startswith("android.permission."):
                         perm_name = perm_name[19:]
                     old_permissions.add(perm_name)
@@ -541,15 +541,15 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
 
                 for permission in sorted_permissions:
                     permel = doc.createElement('uses-permission')
-                    permel.setAttribute('name', permission.name)
+                    permel.setAttribute('name', permission[0])
-                    if permission.maxSdkVersion is not None:
+                    if permission[1] is not None:
-                        permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
+                        permel.setAttribute('maxSdkVersion', '%d' % permission[1])
                         apkel.appendChild(permel)
                 for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
                     permel = doc.createElement('uses-permission-sdk-23')
-                    permel.setAttribute('name', permission_sdk_23.name)
+                    permel.setAttribute('name', permission_sdk_23[0])
-                    if permission_sdk_23.maxSdkVersion is not None:
+                    if permission_sdk_23[1] is not None:
-                        permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
+                        permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23[1])
                         apkel.appendChild(permel)
                 if 'nativecode' in apk:
                     addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)

fdroidserver/update.py

@@ -27,7 +27,7 @@ import re
 import socket
 import zipfile
 import hashlib
-import pickle  # nosec TODO
+import json
 import time
 import copy
 from datetime import datetime
@@ -46,7 +46,7 @@ from . import metadata
 from .common import SdkToolsPopen
 from .exception import BuildException, FDroidException
 
-METADATA_VERSION = 19
+METADATA_VERSION = 20
 
 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
 UNSET_VERSION_CODE = -0x100000000
@@ -56,8 +56,7 @@ APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
 APK_LABEL_ICON_PAT = re.compile(r".*\s+label='(.*)'\s+icon='(.*?)'")
 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
-APK_PERMISSION_PAT = \
+APK_PERMISSION_PAT = re.compile(".*(name='(.*)')(.*maxSdkVersion='(.*)')?.*")
-    re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
 
 screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
@@ -438,7 +437,7 @@ def getsig(apkpath):
 
 
 def get_cache_file():
-    return os.path.join('tmp', 'apkcache')
+    return os.path.join('tmp', 'apkcache.json')
 
 
 def get_cache():
@@ -460,27 +459,46 @@ def get_cache():
     apkcachefile = get_cache_file()
     ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
     if not options.clean and os.path.exists(apkcachefile):
-        with open(apkcachefile, 'rb') as cf:
+        with open(apkcachefile) as fp:
-            apkcache = pickle.load(cf, encoding='utf-8')  # nosec TODO
+            apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict)
         if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
            or apkcache.get('allow_disabled_algorithms') != ada:
-            apkcache = {}
+            apkcache = collections.OrderedDict()
     else:
-        apkcache = {}
+        apkcache = collections.OrderedDict()
 
     apkcache["METADATA_VERSION"] = METADATA_VERSION
     apkcache['allow_disabled_algorithms'] = ada
 
+    for k, v in apkcache.items():
+        if not isinstance(v, dict):
+            continue
+        if 'antiFeatures' in v:
+            v['antiFeatures'] = set(v['antiFeatures'])
+        if 'added' in v:
+            v['added'] = datetime.fromtimestamp(v['added'])
+
     return apkcache
 
 
 def write_cache(apkcache):
+    class Encoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, set):
+                return ['SET'] + list(obj)
+            elif isinstance(obj, datetime):
+                return obj.timestamp()
+            return super().default(obj)
+
     apkcachefile = get_cache_file()
     cache_path = os.path.dirname(apkcachefile)
     if not os.path.exists(cache_path):
         os.makedirs(cache_path)
-    with open(apkcachefile, 'wb') as cf:
+    for k, v in apkcache.items():
-        pickle.dump(apkcache, cf)
+        if isinstance(k, bytes):
+            print('BYTES: ' + str(k) + ' ' + str(v))
+    with open(apkcachefile, 'w') as fp:
+        json.dump(apkcache, fp, cls=Encoder, indent=2)
 
 
 def get_icon_bytes(apkzip, iconsrc):
@@ -948,16 +966,16 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 
     cachechanged = False
     repo_files = []
-    repodir = repodir.encode('utf-8')
+    repodir = repodir.encode()
     for name in os.listdir(repodir):
         file_extension = common.get_file_extension(name)
         if file_extension == 'apk' or file_extension == 'obb':
             continue
         filename = os.path.join(repodir, name)
-        name_utf8 = name.decode('utf-8')
+        name_utf8 = name.decode()
         if filename.endswith(b'_src.tar.gz'):
             logging.debug(_('skipping source tarball: {path}')
-                          .format(path=filename.decode('utf-8')))
+                          .format(path=filename.decode()))
             continue
         if not common.is_repo_file(filename):
             continue
@@ -968,15 +986,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 
         shasum = sha256sum(filename)
         usecache = False
-        if name in apkcache:
+        if name_utf8 in apkcache:
-            repo_file = apkcache[name]
+            repo_file = apkcache[name_utf8]
-            # added time is cached as tuple but used here as datetime instance
-            if 'added' in repo_file:
-                a = repo_file['added']
-                if isinstance(a, datetime):
-                    repo_file['added'] = a
-                else:
-                    repo_file['added'] = datetime(*a[:6])
             if repo_file.get('hash') == shasum:
                 logging.debug(_("Reading {apkfilename} from cache")
                               .format(apkfilename=name_utf8))
@@ -1004,10 +1015,10 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
                 repo_file['versionCode'] = int(m.group(2))
             srcfilename = name + b'_src.tar.gz'
             if os.path.exists(os.path.join(repodir, srcfilename)):
-                repo_file['srcname'] = srcfilename.decode('utf-8')
+                repo_file['srcname'] = srcfilename.decode()
             repo_file['size'] = stat.st_size
 
-            apkcache[name] = repo_file
+            apkcache[name_utf8] = repo_file
             cachechanged = True
 
         if use_date_from_file:

tests/repo/index-v1.json

@@ -1,7 +1,7 @@
 {
   "repo": {
     "timestamp": 1502845383782,
-    "version": 19,
+    "version": 20,
     "name": "My First F-Droid Repo Demo",
     "icon": "fdroid-icon.png",
     "address": "https://MyFirstFDroidRepo.org/fdroid/repo",

tests/repo/index.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <fdroid>
-	<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="19">
+	<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="20">
 		<description>This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/u/fdroid. </description>
 		<mirror>http://foobarfoobarfoobar.onion/fdroid/repo</mirror>
 		<mirror>https://foo.bar/fdroid/repo</mirror>

tests/run-tests

@@ -177,8 +177,8 @@ if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then
     test -e repo/index.xml
     test -e repo/index.jar
     test -e repo/index-v1.jar
-    test -e tmp/apkcache
+    test -e tmp/apkcache.json
-    ! test -z tmp/apkcache
+    ! test -z tmp/apkcache.json
     test -L urzip.apk
     grep -F '<application id=' repo/index.xml > /dev/null
 fi
@@ -808,8 +808,8 @@ else
     test -e repo/index.xml
     test -e repo/index.jar
     test -e repo/index-v1.jar
-    test -e tmp/apkcache
+    test -e tmp/apkcache.json
-    ! test -z tmp/apkcache
+    ! test -z tmp/apkcache.json
     export ANDROID_HOME=$STORED_ANDROID_HOME
 fi
 
@@ -860,8 +860,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -890,8 +890,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -917,8 +917,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -1009,8 +1009,8 @@ $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
 test -e repo/index-v1.jar
-test -e tmp/apkcache
+test -e tmp/apkcache.json
-! test -z tmp/apkcache
+! test -z tmp/apkcache.json
 grep -F '<application id=' repo/index.xml > /dev/null
 
 # now set fake repo_keyalias

tests/update.TestCase

@@ -3,6 +3,7 @@
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
 import git
+import glob
 import inspect
 import logging
 import optparse
@@ -289,6 +290,45 @@ class UpdateTest(unittest.TestCase):
                 self.assertIsNone(apk.get('obbMainFile'))
                 self.assertIsNone(apk.get('obbPatchFile'))
 
+    def test_apkcache_json(self):
+        """test the migration from pickle to json"""
+        os.chdir(os.path.join(localmodule, 'tests'))
+        if os.path.basename(os.getcwd()) != 'tests':
+            raise Exception('This test must be run in the "tests/" subdir')
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        config['ndk_paths'] = dict()
+        config['accepted_formats'] = ['json', 'txt', 'yml']
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+
+        fdroidserver.update.options = type('', (), {})()
+        fdroidserver.update.options.clean = True
+        fdroidserver.update.options.delete_unknown = True
+        fdroidserver.update.options.rename_apks = False
+        fdroidserver.update.options.allow_disabled_algorithms = False
+
+        fdroidserver.metadata.read_metadata(xref=True)
+        knownapks = fdroidserver.common.KnownApks()
+        apkcache = fdroidserver.update.get_cache()
+        self.assertEqual(2, len(apkcache))
+        self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"])
+        self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms,
+                         apkcache['allow_disabled_algorithms'])
+        apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False)
+        fdroidserver.update.write_cache(apkcache)
+
+        fdroidserver.update.options.clean = False
+        read_from_json = fdroidserver.update.get_cache()
+        self.assertEqual(16, len(read_from_json))
+        for f in glob.glob('repo/*.apk'):
+            self.assertTrue(os.path.basename(f) in read_from_json)
+
+        fdroidserver.update.options.clean = True
+        reset = fdroidserver.update.get_cache()
+        self.assertEqual(2, len(reset))
+
     def test_scan_apk(self):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)