From ab145de6bc50568b185827acb75931ea2b1a7a02 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 22 Jul 2015 18:40:31 -0700 Subject: [PATCH] support app metadata in XML format While the current text metadata format is good for human readability and editability, it is difficult to produce and parse using code. XML is a widespread standard format for easy automatic parsing and creating, while having decent human readability. The .pickle for testing is a lightly edited version of the real metadata for net.osmand.plus: * comments were removed * "NonFreeNet" was added as an AntiFeature --- fdroidserver/metadata.py | 100 +++++- tests/metadata.TestCase | 2 +- tests/metadata/net.osmand.plus.pickle | 491 ++++++++++++++++++++++++++ tests/metadata/net.osmand.plus.xml | 180 ++++++++++ 4 files changed, 766 insertions(+), 7 deletions(-) create mode 100644 tests/metadata/net.osmand.plus.pickle create mode 100644 tests/metadata/net.osmand.plus.xml diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 2b75389f..3a389ec2 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -20,10 +20,14 @@ import json import os import re +import sys import glob import cgi import logging +# use the C implementation when available +import xml.etree.cElementTree as ElementTree + from collections import OrderedDict import common @@ -79,6 +83,8 @@ app_defaults = OrderedDict([ # In the order in which they are laid out on files # Sorted by their action and their place in the build timeline +# These variables can have varying datatypes. For example, anything with +# flagtype(v) == 'list' is inited as False, then set as a list of strings. flag_defaults = OrderedDict([ ('disable', False), ('commit', None), @@ -494,6 +500,11 @@ def read_metadata(xref=True): check_metadata(appinfo) apps[appid] = appinfo + for metafile in sorted(glob.glob(os.path.join('metadata', '*.xml'))): + appid, appinfo = parse_xml_metadata(metafile) + check_metadata(appinfo) + apps[appid] = appinfo + if xref: # Parse all descriptions at load time, just to ensure cross-referencing # errors are caught early rather than when they hit the build server. @@ -579,6 +590,20 @@ def get_default_app_info_list(): def post_metadata_parse(thisinfo): + for build in thisinfo['builds']: + for k, v in build.iteritems(): + if k == 'versionCode': + build['vercode'] = str(v) + del build['versionCode'] + elif k == 'versionName': + build['version'] = str(v) + del build['versionName'] + elif flagtype(k) == 'bool': + if v == 'no': + build[k] = False + else: + build[k] = True + if not thisinfo['Description']: thisinfo['Description'].append('No description available') @@ -682,12 +707,6 @@ def parse_json_metadata(metafile): build[k] = ['yes'] else: build[k] = ['no'] - elif k == 'versionCode': - build['vercode'] = v - del build['versionCode'] - elif k == 'versionName': - build['version'] = v - del build['versionName'] # TODO create schema using https://pypi.python.org/pypi/jsonschema post_metadata_parse(thisinfo) @@ -695,6 +714,75 @@ def parse_json_metadata(metafile): return (appid, thisinfo) +def parse_xml_metadata(metafile): + + appid = os.path.basename(metafile)[0:-4] # strip path and .xml + thisinfo = get_default_app_info_list() + thisinfo['id'] = appid + + tree = ElementTree.ElementTree(file=metafile) + root = tree.getroot() + + if root.tag != 'resources': + logging.critical(metafile + ' does not have root as !') + sys.exit(1) + + supported_metadata = app_defaults.keys() + for child in root: + if child.tag != 'builds': + # builds does not have name="" attrib + name = child.attrib['name'] + if name not in supported_metadata: + raise MetaDataException("Unrecognised metadata: <" + + child.tag + ' name="' + name + '">' + + child.text + + "') + + if child.tag == 'string': + thisinfo[name] = child.text + elif child.tag == 'string-array': + items = [] + for item in child: + items.append(item.text) + thisinfo[name] = items + elif child.tag == 'builds': + builds = [] + for build in child: + builddict = dict() + for key in build: + builddict[key.tag] = key.text + builds.append(builddict) + thisinfo['builds'] = builds + + # convert to the odd internal format + for k in ('Description', 'Maintainer Notes'): + if isinstance(thisinfo[k], basestring): + text = thisinfo[k].rstrip().lstrip() + thisinfo[k] = text.split('\n') + + supported_flags = flag_defaults.keys() + ['versionCode', 'versionName'] + for build in thisinfo['builds']: + for k, v in build.iteritems(): + if k not in supported_flags: + raise MetaDataException("Unrecognised build flag: {0}={1}" + .format(k, v)) + keyflagtype = flagtype(k) + if keyflagtype == 'bool': + # TODO handle this using + + + + Tracking + NonFreeNet + + + + Navigation + + + GPLv3 + http://osmand.net + https://github.com/osmandapp/Osmand + https://github.com/osmandapp/Osmand/issues + https://code.google.com/p/osmand/#Please_support_the_project + + OsmAnd~ + Offline/online maps and navigation + Osmand~'s features can be extended by enabling the plugins via the settings, +which include online maps from many sources, tracking, OpenStreetMap (OSM) editing and +accessibility enhancements. + +Map data of both vector and raster types can be stored on the phone memory +card for offline usage, and navigation by default uses offline methods. Map +data packages for many territories can be downloaded from within the app and +there is a desktop program available on the website as well for creating your +own. + +Anti-Features: Tracking - It will send your device and application specs to an +Analytics server upon downloading the list of maps you can download. + +[https://osmandapp.github.io/changes.html Changelog] + + + git + https://github.com/mvdan/OsmAnd-submodules + + + + + + + + 182 + 1.8.2 + 76ada6c8a08afe69acb755503373ac36328ef665 + android/OsmAnd + true + bin/OsmAnd-release-unsigned.apk + sed -i 's/"OsmAnd+"/"OsmAnd~"/g' build.xml + ./old-ndk-build.sh && ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release + no + + + + 1.8.3 + 183 + 1.8.3 + android/OsmAnd + true + bin/OsmAnd-release-unsigned.apk + ../../build + no + + + + 1.9.4 + 196 + 1.9.4 + android/OsmAnd + true + bin/OsmAnd-release-unsigned.apk + ../../build + no + r10d + + + + 1.9.5 + 197 + 1.9.5 + android/OsmAnd + true + bin/OsmAnd-release-unsigned.apk + ../../build + no + r10d + + + + + +No UCMs apply because git never contains actual releases, only pre-releses. + +The build instructions have been moved to a script in the root of the repo, +'build'. This way it can be updated along with the submodules. + + + None + None + 1.9.5 + 197 + + \ No newline at end of file