From 5faef55d67788331f362b42f6a5a8e5cd509a055 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 13 Oct 2016 16:50:31 +0200 Subject: [PATCH 1/5] support all valid versionCode values, i.e. Java Integer values versionCode can be any Java Integer value, from Integer.MAX_VALUE (2147483648) to Integer.MIN_VALUE (-2147483647) --- fdroidserver/update.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 7e1181d1..d18f0fd3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1444,12 +1444,15 @@ def main(): else: archapks = [] + # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE + UNSET_VERSION_CODE = -0x100000000 + # Some information from the apks needs to be applied up to the application # level. When doing this, we use the info from the most recent version's apk. # We deal with figuring out when the app was added and last updated at the # same time. for appid, app in apps.items(): - bestver = 0 + bestver = UNSET_VERSION_CODE for apk in apks + archapks: if apk['id'] == appid: if apk['versioncode'] > bestver: @@ -1467,7 +1470,7 @@ def main(): if not app.lastupdated: logging.debug("Don't know when " + appid + " was last updated") - if bestver == 0: + if bestver == UNSET_VERSION_CODE: if app.Name is None: app.Name = app.AutoName or appid app.icon = None From 47d9fd330d0d1a0344465c2c8be1048ee76b3784 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 13 Oct 2016 17:28:54 +0200 Subject: [PATCH 2/5] remove unused 'apps' argument from update.scan_apks() --- fdroidserver/update.py | 7 +++---- tests/update.TestCase | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index d18f0fd3..a93d982e 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -504,12 +504,11 @@ def insert_obbs(repodir, apps, apks): break -def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False): +def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): """Scan the apks in the given repo directory. This also extracts the icons. - :param apps: list of all applications, as per metadata.read_metadata :param apkcache: current apk cache information :param repodir: repo directory to scan :param knownapks: known apks info @@ -1394,7 +1393,7 @@ def main(): delete_disabled_builds(apps, apkcache, repodirs) # Scan all apks in the main repo - apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk) + apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk) # Generate warnings for apk's with no metadata (or create skeleton # metadata files, if requested on the command line) @@ -1438,7 +1437,7 @@ def main(): # Scan the archive repo for apks as well if len(repodirs) > 1: - archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk) + archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk) if cc: cachechanged = True else: diff --git a/tests/update.TestCase b/tests/update.TestCase index e2927932..25e59a83 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -101,7 +101,7 @@ class UpdateTest(unittest.TestCase): apps = fdroidserver.metadata.read_metadata(xref=True) knownapks = fdroidserver.common.KnownApks() - apks, cachechanged = fdroidserver.update.scan_apks(apps, {}, 'repo', knownapks, False) + apks, cachechanged = fdroidserver.update.scan_apks({}, 'repo', knownapks, False) self.assertEqual(len(apks), 6) apk = apks[0] self.assertEqual(apk['minSdkVersion'], '4') From 36a585c2fcb27a897ec71601a309125ce68e306b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 13 Oct 2016 18:02:44 +0200 Subject: [PATCH 3/5] create addElementIfInApk() function for clean up common operation There are currently a couple different ways this is done in the code, this commit changes all of them to be like addElementNonEmpty(). --- fdroidserver/update.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index a93d982e..09353256 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -883,6 +883,12 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): return addElement(name, value, doc, parent) + def addElementIfInApk(name, apk, key, doc, parent): + if key not in apk: + return + value = str(apk[key]) + addElement(name, value, doc, parent) + def addElementCDATA(name, value, doc, parent): el = doc.createElement(name) el.appendChild(doc.createCDATASection(value)) @@ -1069,8 +1075,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): addElement('version', apk['version'], doc, apkel) addElement('versioncode', str(apk['versioncode']), doc, apkel) addElement('apkname', apk['apkname'], doc, apkel) - if 'srcname' in apk: - addElement('srcname', apk['srcname'], doc, apkel) + addElementIfInApk('srcname', apk, 'srcname', doc, apkel) for hash_type in ['sha256']: if hash_type not in apk: continue @@ -1081,14 +1086,18 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): addElement('sig', apk['sig'], doc, apkel) addElement('size', str(apk['size']), doc, apkel) addElement('sdkver', str(apk['minSdkVersion']), doc, apkel) - if 'targetSdkVersion' in apk: - addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel) - if 'maxSdkVersion' in apk: - addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel) - addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel) - addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel) - addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel) - addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel) + addElementIfInApk('targetSdkVersion', apk, + 'targetSdkVersion', doc, apkel) + addElementIfInApk('maxsdkver', apk, + 'maxSdkVersion', doc, apkel) + addElementIfInApk('obbMainFile', apk, + 'obbMainFile', doc, apkel) + addElementIfInApk('obbMainFileSha256', apk, + 'obbMainFileSha256', doc, apkel) + addElementIfInApk('obbPatchFile', apk, + 'obbPatchFile', doc, apkel) + addElementIfInApk('obbPatchFileSha256', apk, + 'obbPatchFileSha256', doc, apkel) if 'added' in apk: addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel) From 07ce9488097c1361c1cc1a515773fb73199550bf Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 13 Oct 2016 18:24:58 +0200 Subject: [PATCH 4/5] support adding arbitrary files to a repo This adds the most basic level of support for including arbitrary files in an F-Droid repository. This is useful for things like including videos, ebooks, update.zip files for ROM updates, and more. The aim is to have this as generic as possible to keep it flexible for unforeseen uses. Code-wise, this is really just a first effort. This area of code has not been touched in a very long time, and the repo parsing is done in a giant function that is not easy to break apart. It should be broken up to more cleanly support arbitrary files. Also remove the TODO line, we've decided to keep the old permission format for now, at least until there is a major overhaul of the index data format. And the issue tracker the proper place for TODOs. --- fdroidserver/common.py | 6 ++ fdroidserver/update.py | 124 +++++++++++++++++++++++++++++++++-------- 2 files changed, 106 insertions(+), 24 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 6c5e3faa..1710c405 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1609,6 +1609,12 @@ class KnownApks: return lst +def get_file_extension(filename): + """get the normalized file extension, can be blank string but never None""" + + return os.path.splitext(filename)[1].lower()[1:] + + def isApkDebuggable(apkfile, config): """Returns True if the given apk file is debuggable diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 09353256..befad784 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -504,6 +504,76 @@ def insert_obbs(repodir, apps, apks): break +def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): + """Scan a repo for all files with an extension except APK/OBB + + :param apkcache: current cached info about all repo files + :param repodir: repo directory to scan + :param knownapks: list of all known files, as per metadata.read_metadata + :param use_date_from_file: use date from file (instead of current date) + for newly added files + """ + + cachechanged = False + repo_files = [] + for name in os.listdir(repodir): + filename = os.path.join(repodir, name) + file_extension = common.get_file_extension(name) + if name in ['index.jar', 'index.xml', 'index.html', 'categories.txt', ]: + continue + if file_extension == 'apk' or file_extension == 'obb': + continue + if not os.path.isfile(filename): + continue + if os.stat(filename).st_size == 0: + logging.error(filename + ' is zero size!') + sys.exit(1) + + shasum = sha256sum(filename) + usecache = False + if name in apkcache: + repo_file = apkcache[name] + if repo_file['sha256'] == shasum: + logging.debug("Reading " + name + " from cache") + usecache = True + else: + logging.debug("Ignoring stale cache data for " + name) + elif not usecache: + logging.debug("Processing " + name) + repo_file = {} + # TODO rename apkname globally to something more generic + repo_file['name'] = name + repo_file['apkname'] = name + repo_file['sha256'] = shasum + repo_file['versioncode'] = 0 + repo_file['version'] = shasum + # the static ID is the SHA256 unless it is set in the metadata + repo_file['id'] = shasum + srcfilename = name + ".src.tar.gz" + if os.path.exists(os.path.join(repodir, srcfilename)): + repo_file['srcname'] = srcfilename + repo_file['size'] = os.path.getsize(filename) + + apkcache[name] = repo_file + cachechanged = True + + if use_date_from_file: + timestamp = os.stat(filename).st_ctime + default_date_param = datetime.fromtimestamp(timestamp).utctimetuple() + else: + default_date_param = None + + # Record in knownapks, getting the added date at the same time.. + added = knownapks.recordapk(repo_file['apkname'], repo_file['id'], + default_date=default_date_param) + if added: + repo_file['added'] = added + + repo_files.append(repo_file) + + return repo_files, cachechanged + + def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): """Scan the apks in the given repo directory. @@ -1064,6 +1134,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): current_version_code = 0 current_version_file = None for apk in apklist: + file_extension = common.get_file_extension(apk['apkname']) # find the APK for the "Current Version" if current_version_code < apk['versioncode']: current_version_code = apk['versioncode'] @@ -1083,7 +1154,6 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): hashel.setAttribute("type", hash_type) hashel.appendChild(doc.createTextNode(apk[hash_type])) apkel.appendChild(hashel) - addElement('sig', apk['sig'], doc, apkel) addElement('size', str(apk['size']), doc, apkel) addElement('sdkver', str(apk['minSdkVersion']), doc, apkel) addElementIfInApk('targetSdkVersion', apk, @@ -1101,30 +1171,32 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): if 'added' in apk: addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel) - # TODO: remove old permission format - old_permissions = set() - for perm in apk['uses-permission']: - perm_name = perm.name - if perm_name.startswith("android.permission."): - perm_name = perm_name[19:] - old_permissions.add(perm_name) - addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel) + if file_extension == 'apk': # sig is required for APKs, but only APKs + addElement('sig', apk['sig'], doc, apkel) - for permission in apk['uses-permission']: - permel = doc.createElement('uses-permission') - permel.setAttribute('name', permission.name) - if permission.maxSdkVersion is not None: - permel.setAttribute('maxSdkVersion', permission.maxSdkVersion) - apkel.appendChild(permel) - for permission_sdk_23 in apk['uses-permission-sdk-23']: - permel = doc.createElement('uses-permission-sdk-23') - permel.setAttribute('name', permission_sdk_23.name) - if permission_sdk_23.maxSdkVersion is not None: - permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion) - apkel.appendChild(permel) - if 'nativecode' in apk: - addElement('nativecode', ','.join(apk['nativecode']), doc, apkel) - addElementNonEmpty('features', ','.join(apk['features']), doc, apkel) + old_permissions = set() + for perm in apk['uses-permission']: + perm_name = perm.name + if perm_name.startswith("android.permission."): + perm_name = perm_name[19:] + old_permissions.add(perm_name) + addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel) + + for permission in apk['uses-permission']: + permel = doc.createElement('uses-permission') + permel.setAttribute('name', permission.name) + if permission.maxSdkVersion is not None: + permel.setAttribute('maxSdkVersion', permission.maxSdkVersion) + apkel.appendChild(permel) + for permission_sdk_23 in apk['uses-permission-sdk-23']: + permel = doc.createElement('uses-permission-sdk-23') + permel.setAttribute('name', permission_sdk_23.name) + if permission_sdk_23.maxSdkVersion is not None: + permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion) + apkel.appendChild(permel) + if 'nativecode' in apk: + addElement('nativecode', ','.join(apk['nativecode']), doc, apkel) + addElementNonEmpty('features', ','.join(apk['features']), doc, apkel) if current_version_file is not None \ and config['make_current_version_link'] \ @@ -1404,6 +1476,10 @@ def main(): # Scan all apks in the main repo apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk) + files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks, + options.use_date_from_apk) + cachechanged = cachechanged or fcachechanged + apks += files # Generate warnings for apk's with no metadata (or create skeleton # metadata files, if requested on the command line) newmetadata = False From 8e45d300208d4eadd104c2791bc93f7d4505d71c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 2 Nov 2016 15:50:34 +0100 Subject: [PATCH 5/5] reuse os.stat() result when checking for non-APK files This should make things a bit more efficient when running on lots of files, unless python was already caching the result... --- fdroidserver/update.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index befad784..30390f82 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -517,15 +517,16 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): cachechanged = False repo_files = [] for name in os.listdir(repodir): - filename = os.path.join(repodir, name) - file_extension = common.get_file_extension(name) if name in ['index.jar', 'index.xml', 'index.html', 'categories.txt', ]: continue + file_extension = common.get_file_extension(name) if file_extension == 'apk' or file_extension == 'obb': continue + filename = os.path.join(repodir, name) if not os.path.isfile(filename): continue - if os.stat(filename).st_size == 0: + stat = os.stat(filename) + if stat.st_size == 0: logging.error(filename + ' is zero size!') sys.exit(1) @@ -552,13 +553,13 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): srcfilename = name + ".src.tar.gz" if os.path.exists(os.path.join(repodir, srcfilename)): repo_file['srcname'] = srcfilename - repo_file['size'] = os.path.getsize(filename) + repo_file['size'] = stat.st_size apkcache[name] = repo_file cachechanged = True if use_date_from_file: - timestamp = os.stat(filename).st_ctime + timestamp = stat.st_ctime default_date_param = datetime.fromtimestamp(timestamp).utctimetuple() else: default_date_param = None