From 2113e290829c6e2fd09487a70dea3cea8b2695ea Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 Oct 2018 22:23:42 +0200 Subject: [PATCH 1/4] throw error on ancient, obsolete BuildVersion/UseBuilt metadata fields --- fdroidserver/metadata.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 273ecb80..ac204664 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -228,16 +228,15 @@ TYPE_LIST = 4 TYPE_SCRIPT = 5 TYPE_MULTILINE = 6 TYPE_BUILD = 7 -TYPE_BUILD_V2 = 8 -TYPE_INT = 9 +TYPE_INT = 8 fieldtypes = { 'Description': TYPE_MULTILINE, 'MaintainerNotes': TYPE_MULTILINE, 'Categories': TYPE_LIST, 'AntiFeatures': TYPE_LIST, - 'BuildVersion': TYPE_BUILD, - 'Build': TYPE_BUILD_V2, + 'Build': TYPE_BUILD, + 'BuildVersion': TYPE_OBSOLETE, 'UseBuilt': TYPE_OBSOLETE, } @@ -1333,7 +1332,7 @@ def parse_txt_metadata(mf, app): f = f.replace(' ', '') ftype = fieldtype(f) - if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]: + if ftype not in [TYPE_BUILD]: add_comments(f) if ftype == TYPE_MULTILINE: mode = 1 @@ -1345,15 +1344,6 @@ def parse_txt_metadata(mf, app): elif ftype == TYPE_LIST: app[f] = split_list_values(v) elif ftype == TYPE_BUILD: - if v.endswith("\\"): - mode = 2 - del buildlines[:] - buildlines.append(v[:-1]) - else: - build = parse_buildline([v]) - app.builds.append(build) - add_comments('build:' + app.builds[-1].versionCode) - elif ftype == TYPE_BUILD_V2: vv = v.split(',') if len(vv) != 2: warn_or_exception(_('Build should have comma-separated ' @@ -1372,7 +1362,9 @@ def parse_txt_metadata(mf, app): del buildlines[:] mode = 3 elif ftype == TYPE_OBSOLETE: - pass # Just throw it away! + warn_or_exception(_("'{field}' in {linedesc} is obsolete, see docs for current fields:") + .format(field=f, linedesc=linedesc) + + '\nhttps://f-droid.org/docs/') else: warn_or_exception(_("Unrecognised field '{field}' in {linedesc}") .format(field=f, linedesc=linedesc)) From 85993eb2f8fa7f1debcabfc13e1aa864be36b590 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 Oct 2018 23:31:24 +0200 Subject: [PATCH 2/4] lint: check fields for proper type, e.g. list vs. string fdroid/fdroidserver#578 --- fdroidserver/lint.py | 24 ++++++++++++++++++++ tests/lint.TestCase | 53 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index bfa380e7..9eaf4b19 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -473,6 +473,29 @@ def check_extlib_dir(apps): yield _("Unused extlib at %s") % os.path.join(dir_path, path) +def check_app_field_types(app): + """Check the fields have valid data types""" + + for field in app.keys(): + v = app.get(field) + t = metadata.fieldtype(field) + if v is None: + continue + elif field == 'builds': + if not isinstance(v, list): + yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!") + .format(appid=app.id, field=field, + type='list', fieldtype=v.__class__.__name__)) + elif t == metadata.TYPE_LIST and not isinstance(v, list): + yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}!'") + .format(appid=app.id, field=field, + type='list', fieldtype=v.__class__.__name__)) + elif t == metadata.TYPE_STRING and not type(v) in (str, bool, dict): + yield(_("{appid}: {field} must be a '{type}', but it is a '{fieldtype}'!") + .format(appid=app.id, field=field, + type='str', fieldtype=v.__class__.__name__)) + + def check_for_unsupported_metadata_files(basedir=""): """Checks whether any non-metadata files are in metadata/""" @@ -538,6 +561,7 @@ def main(): continue app_check_funcs = [ + check_app_field_types, check_regexes, check_update_check_data_url, check_vercode_operation, diff --git a/tests/lint.TestCase b/tests/lint.TestCase index 8255957c..2670c3dd 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -70,6 +70,59 @@ class LintTest(unittest.TestCase): logging.debug(warn) self.assertTrue(anywarns) + def test_check_app_field_types(self): + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.lint.config = config + + app = fdroidserver.metadata.App() + app.id = 'fake.app' + app.Name = 'Bad App' + app.Summary = 'We pwn you' + app.Description = 'These are some back' + + fields = { + 'AntiFeatures': { + 'good': [ + ['KnownVuln', ], + ['NonFreeNet', 'KnownVuln'], + ], + 'bad': [ + 'KnownVuln', + 'NonFreeNet,KnownVuln', + ], + }, + 'Categories': { + 'good': [ + ['Sports & Health', ], + ['Multimedia', 'Graphics'], + ], + 'bad': [ + 'Science & Education', + 'Multimedia,Graphics', + ], + }, + } + + for field, values in fields.items(): + + for bad in values['bad']: + anywarns = False + app[field] = bad + for warn in fdroidserver.lint.check_app_field_types(app): + anywarns = True + logging.debug(warn) + self.assertTrue(anywarns) + + for good in values['good']: + anywarns = False + app[field] = good + for warn in fdroidserver.lint.check_app_field_types(app): + anywarns = True + logging.debug(warn) + self.assertFalse(anywarns) + def test_check_vercode_operation(self): config = dict() fdroidserver.common.fill_config_defaults(config) From 6e6afeec6599f3ec5c8b9d7e0a9b24219b4cbdd9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 Oct 2018 23:39:09 +0200 Subject: [PATCH 3/4] unset Categories should be empty by default, not ['None'] This is an ancient, bizarre vestige. This makes the TYPE_LIST fields behave the same, e.g. AntiFeatures and Categories. --- fdroidserver/metadata.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ac204664..88b7451f 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -156,7 +156,7 @@ class App(dict): self.Disabled = None self.AntiFeatures = [] self.Provides = None - self.Categories = ['None'] + self.Categories = [] self.License = 'Unknown' self.AuthorName = None self.AuthorEmail = None @@ -903,8 +903,6 @@ def post_metadata_parse(app): if isinstance(app.Categories, str): app.Categories = [app.Categories] - elif app.Categories is None: - app.Categories = ['None'] else: app.Categories = [str(i) for i in app.Categories] From 58b14279af8c49f880fdfb87e2083a3121047028 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 9 Oct 2018 23:49:27 +0200 Subject: [PATCH 4/4] rewritemeta: rewrite AntiFeature str entries like Categories --- fdroidserver/metadata.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 88b7451f..6a1869a7 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -901,10 +901,14 @@ def post_metadata_parse(app): if 'flavours' in app and app['flavours'] == [True]: app['flavours'] = 'yes' - if isinstance(app.Categories, str): - app.Categories = [app.Categories] - else: - app.Categories = [str(i) for i in app.Categories] + for field, fieldtype in fieldtypes.items(): + if fieldtype != TYPE_LIST: + continue + value = app.get(field) + if isinstance(value, str): + app[field] = [value, ] + elif value is not None: + app[field] = [str(i) for i in value] def _yaml_bool_unmapable(v): return v in (True, False, [True], [False])