From c469f0feed29fa0237ef24247f7c7d0454d93c78 Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Tue, 10 Jan 2012 18:57:07 +0000 Subject: [PATCH] Improved metadata handling Two main points: Firstly, there is no longer a random mapping between app['key'] and a corresponding field in the metadata file - the key and field are now the same. Secondly, more information (including comments) is retrieved from the metadata, to facilitate being able to re-write it which is necessary for various support utilities. --- build.py | 8 +- checkmarket2.py | 2 +- common.py | 200 +++++++++++++++++++++++++++++------------------- scanner.py | 4 +- update.py | 60 +++++++-------- 5 files changed, 157 insertions(+), 117 deletions(-) diff --git a/build.py b/build.py index fadb7a54..b69020b5 100755 --- a/build.py +++ b/build.py @@ -74,13 +74,13 @@ if not os.path.isdir(build_dir): for app in apps: - if app['disabled']: + if app['Disabled']: print "Skipping %s: disabled" % app['id'] elif not app['builds']: print "Skipping %s: no builds specified" % app['id'] - if (app['disabled'] is None and app['repo'] != '' - and app['repotype'] != '' and (options.package is None or + if (app['Disabled'] is None and app['Repo'] != '' + and app['Repo Type'] != '' and (options.package is None or options.package == app['id']) and len(app['builds']) > 0): print "Processing " + app['id'] @@ -88,7 +88,7 @@ for app in apps: build_dir = 'build/' + app['id'] # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['repotype'], app['repo'], build_dir) + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) refreshed_source = False diff --git a/checkmarket2.py b/checkmarket2.py index 193a63fa..f1a7d805 100755 --- a/checkmarket2.py +++ b/checkmarket2.py @@ -63,7 +63,7 @@ for app in apps: print "...couldn't find version code" elif not version: print "...couldn't find version" - elif vercode == app['marketvercode'] and version == app['marketversion']: + elif vercode == app['Market Version Code'] and version == app['Market Version']: print "...up to date" else: print '...updating to version:' + version + ' vercode:' + vercode diff --git a/common.py b/common.py index d54ed432..d1bd6170 100644 --- a/common.py +++ b/common.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # common.py - part of the FDroid server tools -# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -245,15 +245,53 @@ class vcs_bzr(vcs): raise VCSException("Bzr update failed") +# Get the type expected for a given metadata field. +def metafieldtype(name): + if name == 'Description': + return 'multiline' + if name == 'Requires Root': + return 'flag' + if name == 'Build Version': + return 'build' + if name == 'Use Built': + return 'obsolete' + return 'string' + +# Parse metadata for a single application. +# +# 'metafile' - the filename to read. The package id for the application comes +# from this filename. +# +# Returns a dictionary containing all the details of the application. There are +# two major kinds of information in the dictionary. Keys beginning with capital +# letters correspond directory to identically named keys in the metadata file. +# Keys beginning with lower case letters are generated in one way or another, +# and are not found verbatim in the metadata. +# +# Known keys not originating from the metadata are: +# +# 'id' - the application's package ID +# 'builds' - a list of dictionaries containing build information +# for each defined build +# 'comments' - a list of comments from the metadata file. Each is +# a tuple of the form (field, comment) where field is +# the name of the field it preceded in the metadata +# file +# 'name' - the application's name, as displayed. This will +# always be None on return from this parser. The name +# is discovered from built APK files. +# def parse_metadata(metafile, **kw): - def parse_buildline(value): + def parse_buildline(lines): + value = "".join(lines) parts = [p.replace("\\,", ",") for p in re.split(r"(? 0: + raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name) + elif fieldtype == 'string': + thisinfo[field] = value + elif fieldtype == 'flag': + if value == 'Yes': + thisinfo[field] = True + elif value == 'No': + thisinfo[field] = False + else: + raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name) + elif fieldtype == 'build': if value.endswith("\\"): mode = 2 - buildline = [value[:-1]] + buildlines = [value[:-1]] else: - thisinfo['builds'].append(parse_buildline(value)) - elif field == "Requires Root": - if value == "Yes": - thisinfo['requiresroot'] = True + thisinfo['builds'].append(parse_buildline([value])) + elif fieldtype == 'obsolete': + pass # Just throw it away! else: - raise MetaDataException("Unrecognised field " + field + " in " + metafile.name) - elif mode == 1: # multi-line description + raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name) + elif mode == 1: # Multiline field if line == '.': mode = 0 else: if len(line) == 0: - thisinfo['description'] += '\n\n' + thisinfo[field] += '\n\n' else: - if (not thisinfo['description'].endswith('\n') and - len(thisinfo['description']) > 0): - thisinfo['description'] += ' ' - thisinfo['description'] += line - elif mode == 2: # line continuation + if (not thisinfo[field].endswith('\n') and + len(thisinfo[field]) > 0): + thisinfo[field] += ' ' + thisinfo[field] += line + elif mode == 2: # Line continuation mode in Build Version if line.endswith("\\"): - buildline.append(line[:-1]) + buildlines.append(line[:-1]) else: - buildline.append(line) + buildlines.append(line) thisinfo['builds'].append( - parse_buildline("".join(buildline))) + parse_buildline(buildlines)) mode = 0 + if mode == 1: - raise MetaDataException("Description not terminated in " + metafile.name) - if len(thisinfo['description']) == 0: - thisinfo['description'] = 'No description available' + raise MetaDataException(field + " not terminated in " + metafile.name) + elif mode == 2: + raise MetaDataException("Unterminated continuation in " + metafile.name) + + if len(thisinfo['Description']) == 0: + thisinfo['Description'] = 'No description available' + + # Ensure all AntiFeatures are recognised... + if thisinfo['AntiFeatures']: + parts = thisinfo['AntiFeatures'].split(",") + for part in parts: + if (part != "Ads" and + part != "Tracking" and + part != "NonFreeNet" and + part != "NonFreeDep" and + part != "NonFreeAdd"): + raise MetaDataException("Unrecognised antifeature '" + part + "' in " \ + + metafile.name) + return thisinfo + def read_metadata(verbose=False): apps = [] for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))): diff --git a/scanner.py b/scanner.py index 3291a2dc..f4d63e35 100755 --- a/scanner.py +++ b/scanner.py @@ -55,7 +55,7 @@ for app in apps: skip = False if options.package and app['id'] != options.package: skip = True - elif app['disabled']: + elif app['Disabled']: print "Skipping %s: disabled" % app['id'] skip = True elif not app['builds']: @@ -71,7 +71,7 @@ for app in apps: build_dir = 'build/' + app['id'] # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['repotype'], app['repo'], build_dir) + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) refreshed_source = False diff --git a/update.py b/update.py index a943bb7e..b9740992 100755 --- a/update.py +++ b/update.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # update.py - part of the FDroid server tools -# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -196,7 +196,7 @@ for app in apps: if app['name'] is None: app['name'] = app['id'] app['icon'] = '' - if app['disabled'] is None: + if app['Disabled'] is None: print "WARNING: Application " + app['id'] + " has no packages" else: if app['name'] is None: @@ -283,14 +283,14 @@ apps_nopkg = 0 for app in apps: - if app['disabled'] is None: + if app['Disabled'] is None: # Get a list of the apks for this app... gotmarketver = False apklist = [] for apk in apks: if apk['id'] == app['id']: - if str(apk['versioncode']) == app['marketvercode']: + if str(apk['versioncode']) == app['Market Version Code']: gotmarketver = True apklist.append(apk) @@ -304,22 +304,22 @@ for app in apps: addElement('id', app['id'], doc, apel) addElement('name', app['name'], doc, apel) - addElement('summary', app['summary'], doc, apel) + addElement('summary', app['Summary'], doc, apel) addElement('icon', app['icon'], doc, apel) - addElement('description', app['description'], doc, apel) - addElement('license', app['license'], doc, apel) - if 'category' in app: - addElement('category', app['category'], doc, apel) - addElement('web', app['web'], doc, apel) - addElement('source', app['source'], doc, apel) - addElement('tracker', app['tracker'], doc, apel) - if app['donate'] != None: - addElement('donate', app['donate'], doc, apel) - addElement('marketversion', app['marketversion'], doc, apel) - addElement('marketvercode', app['marketvercode'], doc, apel) - if not (app['antifeatures'] is None): - addElement('antifeatures', app['antifeatures'], doc, apel) - if app['requiresroot']: + addElement('description', app['Description'], doc, apel) + addElement('license', app['License'], doc, apel) + if 'Category' in app: + addElement('category', app['Category'], doc, apel) + addElement('web', app['Web Site'], doc, apel) + addElement('source', app['Source Code'], doc, apel) + addElement('tracker', app['Issue Tracker'], doc, apel) + if app['Donate'] != None: + addElement('donate', app['Donate'], doc, apel) + addElement('marketversion', app['Market Version'], doc, apel) + addElement('marketvercode', app['Market Version Code'], doc, apel) + if not (app['AntiFeatures'] is None): + addElement('antifeatures', app['AntiFeatures'], doc, apel) + if app['Requires Root']: addElement('requirements', 'root', doc, apel) # Sort the apk list into version order, just so the web site @@ -370,38 +370,38 @@ for app in apps: if options.buildreport: if len(app['builds']) == 0: print ("WARNING: No builds defined for " + app['id'] + - " Source: " + app['source']) + " Source: " + app['Source Code']) warnings += 1 else: - if app['marketvercode'] != '0': + if app['Market Version Code'] != '0': gotbuild = False for build in app['builds']: - if build['vercode'] == app['marketvercode']: + if build['vercode'] == app['Market Version Code']: gotbuild = True if not gotbuild: print ("WARNING: No build data for market version of " - + app['id'] + " (" + app['marketversion'] - + ") " + app['source']) + + app['id'] + " (" + app['Market Version'] + + ") " + app['Source Code']) warnings += 1 # If we don't have the market version, check if there is a build # with a commit ID starting with '!' - this means we can't build it # for some reason, and don't want hassling about it... - if not gotmarketver and app['marketvercode'] != '0': + if not gotmarketver and app['Market Version Code'] != '0': for build in app['builds']: - if build['vercode'] == app['marketvercode']: + if build['vercode'] == app['Market Version Code']: gotmarketver = True # Output a message of harassment if we don't have the market version: - if not gotmarketver and app['marketvercode'] != '0': - addr = app['source'] - print "WARNING: Don't have market version (" + app['marketversion'] + ") of " + app['name'] + if not gotmarketver and app['Market Version Code'] != '0': + addr = app['Source Code'] + print "WARNING: Don't have market version (" + app['Market Version'] + ") of " + app['name'] print " (" + app['id'] + ") " + addr warnings += 1 if options.verbose: # A bit of extra debug info, basically for diagnosing # app developer mistakes: - print " Market vercode:" + app['marketvercode'] + print " Market vercode:" + app['Market Version Code'] print " Got:" for apk in apks: if apk['id'] == app['id']: