From 6c2cf2ccdd7130a034e50f9bea481351d475a272 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 29 Nov 2016 13:40:21 +0100 Subject: [PATCH] convert internal dict keys to match fdroidclient's Apk class This syncs up the field names between the fdroiddata .yml files, the keys used in the implementation in fdroidserver, the index data format, and the final data structures in fdroidclient. This makes it easier for devs to follow, and makes the Jackson parsing library automatically handle converting the data from the index file to Java instances. This bumps the metadata version since the apkcache will have to be discarded. Here are the name changes: * apkname --> apkName * id --> packageName * sha256 --> hash * version --> versionName * versioncode --> versionCode tests/repo/index.xml was changed only to bump the metadata version from 17 to 18. --- examples/config.py | 2 +- fdroidserver/build.py | 2 +- fdroidserver/install.py | 2 +- fdroidserver/publish.py | 2 +- fdroidserver/scanner.py | 2 +- fdroidserver/update.py | 153 ++++++++++++++++++++-------------------- fdroidserver/verify.py | 2 +- tests/update.TestCase | 14 ++-- 8 files changed, 90 insertions(+), 89 deletions(-) diff --git a/examples/config.py b/examples/config.py index 4614e9ca..615cd14e 100644 --- a/examples/config.py +++ b/examples/config.py @@ -84,7 +84,7 @@ The repository of older versions of applications from the main demo repository. # By default, the "current version" link will be based on the "Name" of the # app from the metadata. You can change it to use a different field from the # metadata here: -# current_version_name_source = 'id' +# current_version_name_source = 'packageName' # Optionally, override home directory for gpg # gpghome = '/home/fdroid/somewhere/else/.gnupg' diff --git a/fdroidserver/build.py b/fdroidserver/build.py index a90f532e..c889e972 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1005,7 +1005,7 @@ def parse_commandline(): parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") + parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]") parser.add_argument("-l", "--latest", action="store_true", default=False, help="Build only the latest version of each package") parser.add_argument("-s", "--stop", action="store_true", default=False, diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 5849acad..394f16de 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -48,7 +48,7 @@ def main(): # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") + parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]") parser.add_argument("-a", "--all", action="store_true", default=False, help="Install all signed applications available") options = parser.parse_args() diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 14942c8a..bf4aa2dc 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -41,7 +41,7 @@ def main(): parser = ArgumentParser(usage="%(prog)s [options] " "[APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") + parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]") metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 8e031fcb..018a64e6 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -251,7 +251,7 @@ def main(): # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") + parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]") metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 913941ba..ece1d2bf 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -151,8 +151,8 @@ def update_wiki(apps, sortedids, apks): cantupdate = False buildfails = False for apk in apks: - if apk['id'] == appid: - if str(apk['versioncode']) == app.CurrentVersionCode: + if apk['packageName'] == appid: + if str(apk['versionCode']) == app.CurrentVersionCode: gotcurrentver = True apklist.append(apk) # Include ones we can't build, as a special case... @@ -161,26 +161,26 @@ def update_wiki(apps, sortedids, apks): if build.versionCode == app.CurrentVersionCode: cantupdate = True # TODO: Nasty: vercode is a string in the build, and an int elsewhere - apklist.append({'versioncode': int(build.versionCode), - 'version': build.versionName, + apklist.append({'versionCode': int(build.versionCode), + 'versionName': build.versionName, 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable), }) else: builtit = False for apk in apklist: - if apk['versioncode'] == int(build.versionCode): + if apk['versionCode'] == int(build.versionCode): builtit = True break if not builtit: buildfails = True - apklist.append({'versioncode': int(build.versionCode), - 'version': build.versionName, + apklist.append({'versionCode': int(build.versionCode), + 'versionName': build.versionName, 'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode), }) if app.CurrentVersionCode == '0': cantupdate = True # Sort with most recent first... - apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True) + apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True) wikidata += "=Versions=\n" if len(apklist) == 0: @@ -198,7 +198,7 @@ def update_wiki(apps, sortedids, apks): wikidata += " (version code " + app.CurrentVersionCode + ").\n\n" validapks = 0 for apk in apklist: - wikidata += "==" + apk['version'] + "==\n" + wikidata += "==" + apk['versionName'] + "==\n" if 'buildproblem' in apk: wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n" @@ -209,7 +209,7 @@ def update_wiki(apps, sortedids, apks): wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n" else: wikidata += "the original developer.\n\n" - wikidata += "Version code: " + str(apk['versioncode']) + '\n' + wikidata += "Version code: " + str(apk['versionCode']) + '\n' wikidata += '\n[[Category:' + wikicat + ']]\n' if len(app.NoSourceSince) > 0: @@ -504,7 +504,7 @@ def insert_obbs(repodir, apps, apks): if not re.match(r'^-?[0-9]+$', chunks[1]): obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ') continue - versioncode = int(chunks[1]) + versionCode = int(chunks[1]) packagename = ".".join(chunks[2:-1]) highestVersionCode = java_Integer_MIN_VALUE @@ -512,18 +512,18 @@ def insert_obbs(repodir, apps, apks): obbWarnDelete(f, "OBB's packagename does not match a supported APK: ") continue for apk in apks: - if packagename == apk['id'] and apk['versioncode'] > highestVersionCode: - highestVersionCode = apk['versioncode'] - if versioncode > highestVersionCode: - obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode) + if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode: + highestVersionCode = apk['versionCode'] + if versionCode > highestVersionCode: + obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode) + ') than any APK: ') continue obbsha256 = sha256sum(f) - obbs.append((packagename, versioncode, obbfile, obbsha256)) + obbs.append((packagename, versionCode, obbfile, obbsha256)) for apk in apks: - for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True): - if versioncode <= apk['versioncode'] and packagename == apk['id']: + for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True): + if versionCode <= apk['versionCode'] and packagename == apk['packageName']: if obbfile.startswith('main.') and 'obbMainFile' not in apk: apk['obbMainFile'] = obbfile apk['obbMainFileSha256'] = obbsha256 @@ -572,7 +572,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_file['added'] = a else: repo_file['added'] = datetime(*a[:6]) - if repo_file['sha256'] == shasum: + if repo_file['hash'] == shasum: logging.debug("Reading " + name + " from cache") usecache = True else: @@ -583,20 +583,21 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_file = {} # TODO rename apkname globally to something more generic repo_file['name'] = name - repo_file['apkname'] = name - repo_file['sha256'] = shasum - repo_file['versioncode'] = 0 - repo_file['version'] = shasum + repo_file['apkName'] = name + repo_file['hash'] = shasum + repo_file['hashType'] = 'sha256' + repo_file['versionCode'] = 0 + repo_file['versionName'] = shasum # the static ID is the SHA256 unless it is set in the metadata - repo_file['id'] = shasum + repo_file['packageName'] = shasum n = name.split('_') if len(n) == 2: packageName = n[0] versionCode = n[1].split('.')[0] if re.match(r'^-?[0-9]+$', versionCode) \ and common.is_valid_package_name(name.split('_')[0]): - repo_file['id'] = packageName - repo_file['versioncode'] = int(versionCode) + repo_file['packageName'] = packageName + repo_file['versionCode'] = int(versionCode) srcfilename = name + "_src.tar.gz" if os.path.exists(os.path.join(repodir, srcfilename)): repo_file['srcname'] = srcfilename @@ -612,7 +613,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): default_date_param = None # Record in knownapks, getting the added date at the same time.. - added = knownapks.recordapk(repo_file['apkname'], repo_file['id'], + added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'], default_date=default_date_param) if added: repo_file['added'] = added @@ -668,7 +669,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): usecache = False if apkfilename in apkcache: apk = apkcache[apkfilename] - if apk['sha256'] == shasum: + if apk['hash'] == shasum: logging.debug("Reading " + apkfilename + " from cache") usecache = True else: @@ -677,8 +678,9 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): if not usecache: logging.debug("Processing " + apkfilename) apk = {} - apk['apkname'] = apkfilename - apk['sha256'] = shasum + apk['apkName'] = apkfilename + apk['hash'] = shasum + apk['hashType'] = 'sha256' srcfilename = apkfilename[:-4] + "_src.tar.gz" if os.path.exists(os.path.join(repodir, srcfilename)): apk['srcname'] = srcfilename @@ -705,9 +707,9 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): for line in p.output.splitlines(): if line.startswith("package:"): try: - apk['id'] = re.match(name_pat, line).group(1) - apk['versioncode'] = int(re.match(vercode_pat, line).group(1)) - apk['version'] = re.match(vername_pat, line).group(1) + apk['packageName'] = re.match(name_pat, line).group(1) + apk['versionCode'] = int(re.match(vercode_pat, line).group(1)) + apk['versionName'] = re.match(vername_pat, line).group(1) except Exception as e: logging.error("Package matching failed: " + str(e)) logging.info("Line was: " + line) @@ -821,8 +823,8 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): + 'sudo date -s "' + str(dt_obj) + '"') iconfilename = "%s.%s.png" % ( - apk['id'], - apk['versioncode']) + apk['packageName'], + apk['versionCode']) # Extract the icon file... empty_densities = [] @@ -937,7 +939,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): default_date_param = None # Record in known apks, getting the added date at the same time.. - added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param) + added = knownapks.recordapk(apk['apkName'], apk['packageName'], default_date=default_date_param) if added: apk['added'] = added @@ -1252,7 +1254,7 @@ def make_index_v0(apps, apks, repodir, repodict): # Get a list of the apks for this app... apklist = [] for apk in apks: - if apk['id'] == appid: + if apk['packageName'] == appid: apklist.append(apk) if len(apklist) == 0: @@ -1309,7 +1311,7 @@ def make_index_v0(apps, apks, repodir, repodict): # Sort the apk list into version order, just so the web site # doesn't have to do any work by default... - apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True) + apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True) if 'antiFeatures' in apklist[0]: app.AntiFeatures.extend(apklist[0]['antiFeatures']) @@ -1318,34 +1320,33 @@ def make_index_v0(apps, apks, repodir, repodict): # Check for duplicates - they will make the client unhappy... for i in range(len(apklist) - 1): - if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']: + if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']: logging.critical("duplicate versions: '%s' - '%s'" % ( - apklist[i]['apkname'], apklist[i + 1]['apkname'])) + apklist[i]['apkName'], apklist[i + 1]['apkName'])) sys.exit(1) current_version_code = 0 current_version_file = None for apk in apklist: - file_extension = common.get_file_extension(apk['apkname']) + file_extension = common.get_file_extension(apk['apkName']) # find the APK for the "Current Version" - if current_version_code < apk['versioncode']: - current_version_code = apk['versioncode'] + if current_version_code < apk['versionCode']: + current_version_code = apk['versionCode'] if current_version_code < int(app.CurrentVersionCode): - current_version_file = apk['apkname'] + current_version_file = apk['apkName'] apkel = doc.createElement("package") apel.appendChild(apkel) - addElement('version', apk['version'], doc, apkel) - addElement('versioncode', str(apk['versioncode']), doc, apkel) - addElement('apkname', apk['apkname'], doc, apkel) + addElement('version', apk['versionName'], doc, apkel) + addElement('versioncode', str(apk['versionCode']), doc, apkel) + addElement('apkname', apk['apkName'], doc, apkel) addElementIfInApk('srcname', apk, 'srcname', doc, apkel) - for hash_type in ['sha256']: - if hash_type not in apk: - continue - hashel = doc.createElement("hash") - hashel.setAttribute("type", hash_type) - hashel.appendChild(doc.createTextNode(apk[hash_type])) - apkel.appendChild(hashel) + + hashel = doc.createElement("hash") + hashel.setAttribute('type', 'sha256') + hashel.appendChild(doc.createTextNode(apk['hash'])) + apkel.appendChild(hashel) + addElement('size', str(apk['size']), doc, apkel) addElementIfInApk('sdkver', apk, 'minSdkVersion', doc, apkel) @@ -1493,11 +1494,11 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi def filter_apk_list_sorted(apk_list): res = [] for apk in apk_list: - if apk['id'] == appid: + if apk['packageName'] == appid: res.append(apk) # Sort the apk list by version code. First is highest/newest. - return sorted(res, key=lambda apk: apk['versioncode'], reverse=True) + return sorted(res, key=lambda apk: apk['versionCode'], reverse=True) def move_file(from_dir, to_dir, filename, ignore_missing): from_path = os.path.join(from_dir, filename) @@ -1513,9 +1514,9 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi apklist = filter_apk_list_sorted(apks) # Move back the ones we don't want. for apk in apklist[keepversions:]: - logging.info("Moving " + apk['apkname'] + " to archive") - move_file(repodir, archivedir, apk['apkname'], False) - move_file(repodir, archivedir, apk['apkname'] + '.asc', True) + logging.info("Moving " + apk['apkName'] + " to archive") + move_file(repodir, archivedir, apk['apkName'], False) + move_file(repodir, archivedir, apk['apkName'] + '.asc', True) for density in all_screen_densities: repo_icon_dir = get_icon_dir(repodir, density) archive_icon_dir = get_icon_dir(archivedir, density) @@ -1531,9 +1532,9 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi archapklist = filter_apk_list_sorted(archapks) # Move forward the ones we want again. for apk in archapklist[:required]: - logging.info("Moving " + apk['apkname'] + " from archive") - move_file(archivedir, repodir, apk['apkname'], False) - move_file(archivedir, repodir, apk['apkname'] + '.asc', True) + logging.info("Moving " + apk['apkName'] + " from archive") + move_file(archivedir, repodir, apk['apkName'], False) + move_file(archivedir, repodir, apk['apkName'] + '.asc', True) for density in all_screen_densities: repo_icon_dir = get_icon_dir(repodir, density) archive_icon_dir = get_icon_dir(archivedir, density) @@ -1549,16 +1550,16 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi def add_apks_to_per_app_repos(repodir, apks): apks_per_app = dict() for apk in apks: - apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid') + apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid') apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo') apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons') - apks_per_app[apk['id']] = apk + apks_per_app[apk['packageName']] = apk if not os.path.exists(apk['per_app_icons']): - logging.info('Adding new repo for only ' + apk['id']) + logging.info('Adding new repo for only ' + apk['packageName']) os.makedirs(apk['per_app_icons']) - apkpath = os.path.join(repodir, apk['apkname']) + apkpath = os.path.join(repodir, apk['apkName']) shutil.copy(apkpath, apk['per_app_repo']) apksigpath = apkpath + '.sig' if os.path.exists(apksigpath): @@ -1766,12 +1767,12 @@ def main(): # metadata files, if requested on the command line) newmetadata = False for apk in apks: - if apk['id'] not in apps: + if apk['packageName'] not in apps: if options.create_metadata: if 'name' not in apk: - logging.error(apk['id'] + ' does not have a name! Skipping...') + logging.error(apk['packageName'] + ' does not have a name! Skipping...') continue - f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8') + f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8') f.write("License:Unknown\n") f.write("Web Site:\n") f.write("Source Code:\n") @@ -1783,13 +1784,13 @@ def main(): f.write(".\n") f.write("Name:" + apk['name'] + "\n") f.close() - logging.info("Generated skeleton metadata for " + apk['id']) + logging.info("Generated skeleton metadata for " + apk['packageName']) newmetadata = True else: - msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!" + msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!" if options.delete_unknown: - logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname']) - rmf = os.path.join(repodirs[0], apk['apkname']) + logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName']) + rmf = os.path.join(repodirs[0], apk['apkName']) if not os.path.exists(rmf): logging.error("Could not find {0} to remove it".format(rmf)) else: @@ -1821,9 +1822,9 @@ def main(): for appid, app in apps.items(): bestver = UNSET_VERSION_CODE for apk in apks + archapks: - if apk['id'] == appid: - if apk['versioncode'] > bestver: - bestver = apk['versioncode'] + if apk['packageName'] == appid: + if apk['versionCode'] > bestver: + bestver = apk['versionCode'] bestapk = apk if 'added' in apk: diff --git a/fdroidserver/verify.py b/fdroidserver/verify.py index c277cf71..c13055a0 100644 --- a/fdroidserver/verify.py +++ b/fdroidserver/verify.py @@ -37,7 +37,7 @@ def main(): # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") + parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]") options = parser.parse_args() config = common.read_config(options) diff --git a/tests/update.TestCase b/tests/update.TestCase index 12adce75..63b1b096 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -110,23 +110,23 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.insert_obbs('repo', apps, apks) for apk in apks: - if apk['id'] == 'obb.mainpatch.current': + if apk['packageName'] == 'obb.mainpatch.current': self.assertEqual(apk.get('obbMainFile'), 'main.1619.obb.mainpatch.current.obb') self.assertEqual(apk.get('obbPatchFile'), 'patch.1619.obb.mainpatch.current.obb') - elif apk['id'] == 'obb.main.oldversion': + elif apk['packageName'] == 'obb.main.oldversion': self.assertEqual(apk.get('obbMainFile'), 'main.1434483388.obb.main.oldversion.obb') self.assertIsNone(apk.get('obbPatchFile')) - elif apk['id'] == 'obb.main.twoversions': + elif apk['packageName'] == 'obb.main.twoversions': self.assertIsNone(apk.get('obbPatchFile')) - if apk['versioncode'] == 1101613: + if apk['versionCode'] == 1101613: self.assertEqual(apk.get('obbMainFile'), 'main.1101613.obb.main.twoversions.obb') - elif apk['versioncode'] == 1101615: + elif apk['versionCode'] == 1101615: self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb') - elif apk['versioncode'] == 1101617: + elif apk['versionCode'] == 1101617: self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb') else: self.assertTrue(False) - elif apk['id'] == 'info.guardianproject.urzip': + elif apk['packageName'] == 'info.guardianproject.urzip': self.assertIsNone(apk.get('obbMainFile')) self.assertIsNone(apk.get('obbPatchFile'))