mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-19 21:30:10 +01:00
Merge branch 'drop-pickle-for-json' into 'master'
Drop pickle for json Closes #163 See merge request fdroid/fdroidserver!568
This commit is contained in:
Hans-Christoph Steiner
commit
8f48976cb9
@ -533,7 +533,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
|
||||
old_permissions = set()
|
||||
sorted_permissions = sorted(apk['uses-permission'])
|
||||
for perm in sorted_permissions:
|
||||
perm_name = perm.name
|
||||
perm_name = perm[0]
|
||||
if perm_name.startswith("android.permission."):
|
||||
perm_name = perm_name[19:]
|
||||
old_permissions.add(perm_name)
|
||||
@ -541,15 +541,15 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
|
||||
|
||||
for permission in sorted_permissions:
|
||||
permel = doc.createElement('uses-permission')
|
||||
permel.setAttribute('name', permission.name)
|
||||
if permission.maxSdkVersion is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
|
||||
permel.setAttribute('name', permission[0])
|
||||
if permission[1] is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission[1])
|
||||
apkel.appendChild(permel)
|
||||
for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
|
||||
permel = doc.createElement('uses-permission-sdk-23')
|
||||
permel.setAttribute('name', permission_sdk_23.name)
|
||||
if permission_sdk_23.maxSdkVersion is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
|
||||
permel.setAttribute('name', permission_sdk_23[0])
|
||||
if permission_sdk_23[1] is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23[1])
|
||||
apkel.appendChild(permel)
|
||||
if 'nativecode' in apk:
|
||||
addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
|
||||
|
||||
@ -27,7 +27,7 @@ import re
|
||||
import socket
|
||||
import zipfile
|
||||
import hashlib
|
||||
import pickle # nosec TODO
|
||||
import json
|
||||
import time
|
||||
import copy
|
||||
from datetime import datetime
|
||||
@ -46,7 +46,7 @@ from . import metadata
|
||||
from .common import SdkToolsPopen
|
||||
from .exception import BuildException, FDroidException
|
||||
|
||||
METADATA_VERSION = 19
|
||||
METADATA_VERSION = 20
|
||||
|
||||
# less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
|
||||
UNSET_VERSION_CODE = -0x100000000
|
||||
@ -56,8 +56,7 @@ APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
|
||||
APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
|
||||
APK_LABEL_ICON_PAT = re.compile(r".*\s+label='(.*)'\s+icon='(.*?)'")
|
||||
APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
|
||||
APK_PERMISSION_PAT = \
|
||||
re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
|
||||
APK_PERMISSION_PAT = re.compile(".*(name='(.*)')(.*maxSdkVersion='(.*)')?.*")
|
||||
APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
|
||||
|
||||
screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
|
||||
@ -438,7 +437,7 @@ def getsig(apkpath):
|
||||
|
||||
|
||||
def get_cache_file():
|
||||
return os.path.join('tmp', 'apkcache')
|
||||
return os.path.join('tmp', 'apkcache.json')
|
||||
|
||||
|
||||
def get_cache():
|
||||
@ -460,27 +459,46 @@ def get_cache():
|
||||
apkcachefile = get_cache_file()
|
||||
ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
|
||||
if not options.clean and os.path.exists(apkcachefile):
|
||||
with open(apkcachefile, 'rb') as cf:
|
||||
apkcache = pickle.load(cf, encoding='utf-8') # nosec TODO
|
||||
with open(apkcachefile) as fp:
|
||||
apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict)
|
||||
if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
|
||||
or apkcache.get('allow_disabled_algorithms') != ada:
|
||||
apkcache = {}
|
||||
apkcache = collections.OrderedDict()
|
||||
else:
|
||||
apkcache = {}
|
||||
apkcache = collections.OrderedDict()
|
||||
|
||||
apkcache["METADATA_VERSION"] = METADATA_VERSION
|
||||
apkcache['allow_disabled_algorithms'] = ada
|
||||
|
||||
for k, v in apkcache.items():
|
||||
if not isinstance(v, dict):
|
||||
continue
|
||||
if 'antiFeatures' in v:
|
||||
v['antiFeatures'] = set(v['antiFeatures'])
|
||||
if 'added' in v:
|
||||
v['added'] = datetime.fromtimestamp(v['added'])
|
||||
|
||||
return apkcache
|
||||
|
||||
|
||||
def write_cache(apkcache):
|
||||
class Encoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, set):
|
||||
return ['SET'] + list(obj)
|
||||
elif isinstance(obj, datetime):
|
||||
return obj.timestamp()
|
||||
return super().default(obj)
|
||||
|
||||
apkcachefile = get_cache_file()
|
||||
cache_path = os.path.dirname(apkcachefile)
|
||||
if not os.path.exists(cache_path):
|
||||
os.makedirs(cache_path)
|
||||
with open(apkcachefile, 'wb') as cf:
|
||||
pickle.dump(apkcache, cf)
|
||||
for k, v in apkcache.items():
|
||||
if isinstance(k, bytes):
|
||||
print('BYTES: ' + str(k) + ' ' + str(v))
|
||||
with open(apkcachefile, 'w') as fp:
|
||||
json.dump(apkcache, fp, cls=Encoder, indent=2)
|
||||
|
||||
|
||||
def get_icon_bytes(apkzip, iconsrc):
|
||||
@ -948,16 +966,16 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
||||
|
||||
cachechanged = False
|
||||
repo_files = []
|
||||
repodir = repodir.encode('utf-8')
|
||||
repodir = repodir.encode()
|
||||
for name in os.listdir(repodir):
|
||||
file_extension = common.get_file_extension(name)
|
||||
if file_extension == 'apk' or file_extension == 'obb':
|
||||
continue
|
||||
filename = os.path.join(repodir, name)
|
||||
name_utf8 = name.decode('utf-8')
|
||||
name_utf8 = name.decode()
|
||||
if filename.endswith(b'_src.tar.gz'):
|
||||
logging.debug(_('skipping source tarball: {path}')
|
||||
.format(path=filename.decode('utf-8')))
|
||||
.format(path=filename.decode()))
|
||||
continue
|
||||
if not common.is_repo_file(filename):
|
||||
continue
|
||||
@ -968,15 +986,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
||||
|
||||
shasum = sha256sum(filename)
|
||||
usecache = False
|
||||
if name in apkcache:
|
||||
repo_file = apkcache[name]
|
||||
# added time is cached as tuple but used here as datetime instance
|
||||
if 'added' in repo_file:
|
||||
a = repo_file['added']
|
||||
if isinstance(a, datetime):
|
||||
repo_file['added'] = a
|
||||
else:
|
||||
repo_file['added'] = datetime(*a[:6])
|
||||
if name_utf8 in apkcache:
|
||||
repo_file = apkcache[name_utf8]
|
||||
if repo_file.get('hash') == shasum:
|
||||
logging.debug(_("Reading {apkfilename} from cache")
|
||||
.format(apkfilename=name_utf8))
|
||||
@ -1004,10 +1015,10 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
||||
repo_file['versionCode'] = int(m.group(2))
|
||||
srcfilename = name + b'_src.tar.gz'
|
||||
if os.path.exists(os.path.join(repodir, srcfilename)):
|
||||
repo_file['srcname'] = srcfilename.decode('utf-8')
|
||||
repo_file['srcname'] = srcfilename.decode()
|
||||
repo_file['size'] = stat.st_size
|
||||
|
||||
apkcache[name] = repo_file
|
||||
apkcache[name_utf8] = repo_file
|
||||
cachechanged = True
|
||||
|
||||
if use_date_from_file:
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"repo": {
|
||||
"timestamp": 1502845383782,
|
||||
"version": 19,
|
||||
"version": 20,
|
||||
"name": "My First F-Droid Repo Demo",
|
||||
"icon": "fdroid-icon.png",
|
||||
"address": "https://MyFirstFDroidRepo.org/fdroid/repo",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<fdroid>
|
||||
<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="19">
|
||||
<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="20">
|
||||
<description>This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/u/fdroid. </description>
|
||||
<mirror>http://foobarfoobarfoobar.onion/fdroid/repo</mirror>
|
||||
<mirror>https://foo.bar/fdroid/repo</mirror>
|
||||
|
||||
@ -177,8 +177,8 @@ if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
test -L urzip.apk
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
fi
|
||||
@ -808,8 +808,8 @@ else
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
export ANDROID_HOME=$STORED_ANDROID_HOME
|
||||
fi
|
||||
|
||||
@ -860,8 +860,8 @@ $fdroid readmeta
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
|
||||
@ -890,8 +890,8 @@ $fdroid readmeta
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
|
||||
@ -917,8 +917,8 @@ $fdroid readmeta
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
|
||||
@ -1009,8 +1009,8 @@ $fdroid readmeta
|
||||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
# now set fake repo_keyalias
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
|
||||
|
||||
import git
|
||||
import glob
|
||||
import inspect
|
||||
import logging
|
||||
import optparse
|
||||
@ -289,6 +290,45 @@ class UpdateTest(unittest.TestCase):
|
||||
self.assertIsNone(apk.get('obbMainFile'))
|
||||
self.assertIsNone(apk.get('obbPatchFile'))
|
||||
|
||||
def test_apkcache_json(self):
|
||||
"""test the migration from pickle to json"""
|
||||
os.chdir(os.path.join(localmodule, 'tests'))
|
||||
if os.path.basename(os.getcwd()) != 'tests':
|
||||
raise Exception('This test must be run in the "tests/" subdir')
|
||||
|
||||
config = dict()
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
config['ndk_paths'] = dict()
|
||||
config['accepted_formats'] = ['json', 'txt', 'yml']
|
||||
fdroidserver.common.config = config
|
||||
fdroidserver.update.config = config
|
||||
|
||||
fdroidserver.update.options = type('', (), {})()
|
||||
fdroidserver.update.options.clean = True
|
||||
fdroidserver.update.options.delete_unknown = True
|
||||
fdroidserver.update.options.rename_apks = False
|
||||
fdroidserver.update.options.allow_disabled_algorithms = False
|
||||
|
||||
fdroidserver.metadata.read_metadata(xref=True)
|
||||
knownapks = fdroidserver.common.KnownApks()
|
||||
apkcache = fdroidserver.update.get_cache()
|
||||
self.assertEqual(2, len(apkcache))
|
||||
self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"])
|
||||
self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms,
|
||||
apkcache['allow_disabled_algorithms'])
|
||||
apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False)
|
||||
fdroidserver.update.write_cache(apkcache)
|
||||
|
||||
fdroidserver.update.options.clean = False
|
||||
read_from_json = fdroidserver.update.get_cache()
|
||||
self.assertEqual(16, len(read_from_json))
|
||||
for f in glob.glob('repo/*.apk'):
|
||||
self.assertTrue(os.path.basename(f) in read_from_json)
|
||||
|
||||
fdroidserver.update.options.clean = True
|
||||
reset = fdroidserver.update.get_cache()
|
||||
self.assertEqual(2, len(reset))
|
||||
|
||||
def test_scan_apk(self):
|
||||
config = dict()
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user