mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-19 03:30:12 +02:00
Merge branch 'metadata-sort-and-refactor' into 'master'
metadata: remove obsolete quirks; add sorting and removal of empty values Closes #332 See merge request fdroid/fdroidserver!1365
This commit is contained in:
commit
cf887583c0
@ -58,9 +58,6 @@ metadata_v0:
|
|||||||
- ../tests/dump_internal_metadata_format.py
|
- ../tests/dump_internal_metadata_format.py
|
||||||
- sed -i
|
- sed -i
|
||||||
-e '/RequiresRoot:/d'
|
-e '/RequiresRoot:/d'
|
||||||
-e "/buildozer/d"
|
|
||||||
-e '/^comments\W /d'
|
|
||||||
-e 's,maven\(\W\) false,maven\1 null,'
|
|
||||||
metadata/dump_*/*.yaml
|
metadata/dump_*/*.yaml
|
||||||
- diff -uw metadata/dump_*
|
- diff -uw metadata/dump_*
|
||||||
|
|
||||||
|
@ -1070,6 +1070,26 @@ def post_parse_yaml_metadata(yamldata):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _format_multiline(value):
|
||||||
|
"""TYPE_MULTILINE with newlines in them are saved as YAML literal strings."""
|
||||||
|
if '\n' in value:
|
||||||
|
return ruamel.yaml.scalarstring.preserve_literal(str(value))
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _format_list(value):
|
||||||
|
"""TYPE_LIST should not contain null values."""
|
||||||
|
return [v for v in value if v]
|
||||||
|
|
||||||
|
|
||||||
|
def _format_script(value):
|
||||||
|
"""TYPE_SCRIPT with one value are converted to YAML string values."""
|
||||||
|
value = [v for v in value if v]
|
||||||
|
if len(value) == 1:
|
||||||
|
return value[0]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _format_stringmap(appid, field, stringmap, versionCode=None):
|
def _format_stringmap(appid, field, stringmap, versionCode=None):
|
||||||
"""Format TYPE_STRINGMAP taking into account localized files in the metadata dir.
|
"""Format TYPE_STRINGMAP taking into account localized files in the metadata dir.
|
||||||
|
|
||||||
@ -1127,7 +1147,7 @@ def _format_stringmap(appid, field, stringmap, versionCode=None):
|
|||||||
make_list = False
|
make_list = False
|
||||||
break
|
break
|
||||||
if make_list:
|
if make_list:
|
||||||
return outlist
|
return sorted(outlist, key=str.lower)
|
||||||
return stringmap
|
return stringmap
|
||||||
|
|
||||||
|
|
||||||
@ -1142,48 +1162,34 @@ def _del_duplicated_NoSourceSince(app):
|
|||||||
del app['AntiFeatures'][key]
|
del app['AntiFeatures'][key]
|
||||||
|
|
||||||
|
|
||||||
def _field_to_yaml(typ, value):
|
|
||||||
"""Convert data to YAML 1.2 format that keeps the right TYPE_*."""
|
|
||||||
if typ == TYPE_STRING:
|
|
||||||
return str(value)
|
|
||||||
elif typ == TYPE_INT:
|
|
||||||
return int(value)
|
|
||||||
elif typ == TYPE_MULTILINE:
|
|
||||||
if '\n' in value:
|
|
||||||
return ruamel.yaml.scalarstring.preserve_literal(str(value))
|
|
||||||
else:
|
|
||||||
return str(value)
|
|
||||||
elif typ == TYPE_SCRIPT:
|
|
||||||
if type(value) == list:
|
|
||||||
if len(value) == 1:
|
|
||||||
return value[0]
|
|
||||||
else:
|
|
||||||
return value
|
|
||||||
else:
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _builds_to_yaml(app):
|
def _builds_to_yaml(app):
|
||||||
|
"""Reformat Builds: flags for output to YAML 1.2.
|
||||||
|
|
||||||
|
This will strip any flag/value that is not set or is empty.
|
||||||
|
TYPE_BOOL fields are removed when they are false. 0 is valid
|
||||||
|
value, it should not be stripped, so there are special cases to
|
||||||
|
handle that.
|
||||||
|
|
||||||
|
"""
|
||||||
builds = ruamel.yaml.comments.CommentedSeq()
|
builds = ruamel.yaml.comments.CommentedSeq()
|
||||||
for build in app.get('Builds', []):
|
for build in app.get('Builds', []):
|
||||||
if not isinstance(build, Build):
|
|
||||||
build = Build(build)
|
|
||||||
b = ruamel.yaml.comments.CommentedMap()
|
b = ruamel.yaml.comments.CommentedMap()
|
||||||
for field in build_flags:
|
for field in build_flags:
|
||||||
if hasattr(build, field):
|
v = build.get(field)
|
||||||
value = getattr(build, field)
|
if v is None or v is False or v == '' or v == dict() or v == list():
|
||||||
if field == 'gradle' and value == ['off']:
|
continue
|
||||||
value = [
|
_flagtype = flagtype(field)
|
||||||
ruamel.yaml.scalarstring.SingleQuotedScalarString('off')
|
if _flagtype == TYPE_MULTILINE:
|
||||||
]
|
v = _format_multiline(v)
|
||||||
typ = flagtype(field)
|
elif _flagtype == TYPE_LIST:
|
||||||
# don't check value == True for TYPE_INT as it could be 0
|
v = _format_list(v)
|
||||||
if value and typ == TYPE_STRINGMAP:
|
elif _flagtype == TYPE_SCRIPT:
|
||||||
v = _format_stringmap(app['id'], field, value, build['versionCode'])
|
v = _format_script(v)
|
||||||
if v:
|
elif _flagtype == TYPE_STRINGMAP:
|
||||||
b[field] = v
|
v = _format_stringmap(app['id'], field, v, build['versionCode'])
|
||||||
elif value is not None and (typ == TYPE_INT or value):
|
|
||||||
b.update({field: _field_to_yaml(typ, value)})
|
if v or v == 0:
|
||||||
|
b[field] = v
|
||||||
|
|
||||||
builds.append(b)
|
builds.append(b)
|
||||||
|
|
||||||
@ -1205,11 +1211,12 @@ def _app_to_yaml(app):
|
|||||||
else:
|
else:
|
||||||
value = app.get(field)
|
value = app.get(field)
|
||||||
if value or field == 'Builds':
|
if value or field == 'Builds':
|
||||||
|
_fieldtype = fieldtype(field)
|
||||||
if field == 'Builds':
|
if field == 'Builds':
|
||||||
if app.get('Builds'):
|
if app.get('Builds'):
|
||||||
cm.update({field: _builds_to_yaml(app)})
|
cm.update({field: _builds_to_yaml(app)})
|
||||||
elif field == 'CurrentVersionCode':
|
elif field == 'Categories':
|
||||||
cm[field] = _field_to_yaml(TYPE_INT, value)
|
cm[field] = sorted(value, key=str.lower)
|
||||||
elif field == 'AntiFeatures':
|
elif field == 'AntiFeatures':
|
||||||
v = _format_stringmap(app['id'], field, value)
|
v = _format_stringmap(app['id'], field, value)
|
||||||
if v:
|
if v:
|
||||||
@ -1217,11 +1224,20 @@ def _app_to_yaml(app):
|
|||||||
elif field == 'AllowedAPKSigningKeys':
|
elif field == 'AllowedAPKSigningKeys':
|
||||||
value = [str(i).lower() for i in value]
|
value = [str(i).lower() for i in value]
|
||||||
if len(value) == 1:
|
if len(value) == 1:
|
||||||
cm[field] = _field_to_yaml(TYPE_STRING, value[0])
|
cm[field] = value[0]
|
||||||
else:
|
else:
|
||||||
cm[field] = _field_to_yaml(TYPE_LIST, value)
|
cm[field] = value
|
||||||
|
elif _fieldtype == TYPE_MULTILINE:
|
||||||
|
v = _format_multiline(value)
|
||||||
|
if v:
|
||||||
|
cm[field] = v
|
||||||
|
elif _fieldtype == TYPE_SCRIPT:
|
||||||
|
v = _format_script(value)
|
||||||
|
if v:
|
||||||
|
cm[field] = v
|
||||||
else:
|
else:
|
||||||
cm[field] = _field_to_yaml(fieldtype(field), value)
|
if value:
|
||||||
|
cm[field] = value
|
||||||
|
|
||||||
if insert_newline:
|
if insert_newline:
|
||||||
# we need to prepend a newline in front of this field
|
# we need to prepend a newline in front of this field
|
||||||
|
@ -717,10 +717,10 @@ class MetadataTest(unittest.TestCase):
|
|||||||
app.UpdateCheckMode = 'Tags'
|
app.UpdateCheckMode = 'Tags'
|
||||||
build = fdroidserver.metadata.Build()
|
build = fdroidserver.metadata.Build()
|
||||||
build.versionName = 'Unknown' # taken from fdroidserver/import.py
|
build.versionName = 'Unknown' # taken from fdroidserver/import.py
|
||||||
build.versionCode = '0' # taken from fdroidserver/import.py
|
build.versionCode = 0 # taken from fdroidserver/import.py
|
||||||
build.disable = 'Generated by import.py ...'
|
build.disable = 'Generated by import.py ...'
|
||||||
build.commit = 'Unknown'
|
build.commit = 'Unknown'
|
||||||
build.gradle = [True]
|
build.gradle = ['yes']
|
||||||
app['Builds'] = [build]
|
app['Builds'] = [build]
|
||||||
|
|
||||||
fdroidserver.metadata.write_yaml(mf, app)
|
fdroidserver.metadata.write_yaml(mf, app)
|
||||||
@ -745,7 +745,7 @@ class MetadataTest(unittest.TestCase):
|
|||||||
disable: Generated by import.py ...
|
disable: Generated by import.py ...
|
||||||
commit: Unknown
|
commit: Unknown
|
||||||
gradle:
|
gradle:
|
||||||
- true
|
- yes
|
||||||
|
|
||||||
AutoUpdateMode: None
|
AutoUpdateMode: None
|
||||||
UpdateCheckMode: Tags
|
UpdateCheckMode: Tags
|
||||||
@ -1886,14 +1886,14 @@ class MetadataTest(unittest.TestCase):
|
|||||||
app = metadata.App({'Builds': [metadata.Build({'rm': []})]})
|
app = metadata.App({'Builds': [metadata.Build({'rm': []})]})
|
||||||
self.assertEqual(dict(), metadata._app_to_yaml(app)['Builds'][0])
|
self.assertEqual(dict(), metadata._app_to_yaml(app)['Builds'][0])
|
||||||
|
|
||||||
def test_app_to_yaml_build_list_string(self):
|
|
||||||
app = metadata.App({'Builds': [metadata.Build({'rm': 'one'})]})
|
|
||||||
self.assertEqual({'rm': 'one'}, metadata._app_to_yaml(app)['Builds'][0])
|
|
||||||
|
|
||||||
def test_app_to_yaml_build_list_one(self):
|
def test_app_to_yaml_build_list_one(self):
|
||||||
app = metadata.App({'Builds': [metadata.Build({'rm': ['one']})]})
|
app = metadata.App({'Builds': [metadata.Build({'rm': ['one']})]})
|
||||||
self.assertEqual({'rm': ['one']}, metadata._app_to_yaml(app)['Builds'][0])
|
self.assertEqual({'rm': ['one']}, metadata._app_to_yaml(app)['Builds'][0])
|
||||||
|
|
||||||
|
def test_app_to_yaml_build_list_two(self):
|
||||||
|
app = metadata.App({'Builds': [metadata.Build({'rm': ['1', '2']})]})
|
||||||
|
self.assertEqual({'rm': ['1', '2']}, metadata._app_to_yaml(app)['Builds'][0])
|
||||||
|
|
||||||
def test_app_to_yaml_build_list(self):
|
def test_app_to_yaml_build_list(self):
|
||||||
app = metadata.App({'Builds': [metadata.Build({'rm': ['b2', 'NO1']})]})
|
app = metadata.App({'Builds': [metadata.Build({'rm': ['b2', 'NO1']})]})
|
||||||
self.assertEqual({'rm': ['b2', 'NO1']}, metadata._app_to_yaml(app)['Builds'][0])
|
self.assertEqual({'rm': ['b2', 'NO1']}, metadata._app_to_yaml(app)['Builds'][0])
|
||||||
@ -1920,6 +1920,53 @@ class MetadataTest(unittest.TestCase):
|
|||||||
cm = metadata._app_to_yaml(metadata.App({'CurrentVersionCode': 0}))
|
cm = metadata._app_to_yaml(metadata.App({'CurrentVersionCode': 0}))
|
||||||
self.assertFalse('CurrentVersionCode' in cm)
|
self.assertFalse('CurrentVersionCode' in cm)
|
||||||
|
|
||||||
|
def test_format_multiline(self):
|
||||||
|
self.assertEqual(metadata._format_multiline('description'), 'description')
|
||||||
|
|
||||||
|
def test_format_multiline_empty(self):
|
||||||
|
self.assertEqual(metadata._format_multiline(''), '')
|
||||||
|
|
||||||
|
def test_format_multiline_newline_char(self):
|
||||||
|
self.assertEqual(metadata._format_multiline('one\\ntwo'), 'one\\ntwo')
|
||||||
|
|
||||||
|
def test_format_multiline_newlines(self):
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._format_multiline(
|
||||||
|
textwrap.dedent(
|
||||||
|
"""
|
||||||
|
one
|
||||||
|
two
|
||||||
|
three
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
),
|
||||||
|
'\none\ntwo\nthree\n',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_format_list_empty(self):
|
||||||
|
self.assertEqual(metadata._format_list(['', None]), list())
|
||||||
|
|
||||||
|
def test_format_list_one_empty(self):
|
||||||
|
self.assertEqual(metadata._format_list(['foo', None]), ['foo'])
|
||||||
|
|
||||||
|
def test_format_list_two(self):
|
||||||
|
self.assertEqual(metadata._format_list(['2', '1']), ['2', '1'])
|
||||||
|
|
||||||
|
def test_format_list_newline(self):
|
||||||
|
self.assertEqual(metadata._format_list(['one\ntwo']), ['one\ntwo'])
|
||||||
|
|
||||||
|
def test_format_list_newline_char(self):
|
||||||
|
self.assertEqual(metadata._format_list(['one\\ntwo']), ['one\\ntwo'])
|
||||||
|
|
||||||
|
def test_format_script_empty(self):
|
||||||
|
self.assertEqual(metadata._format_script(['', None]), list())
|
||||||
|
|
||||||
|
def test_format_script_newline(self):
|
||||||
|
self.assertEqual(metadata._format_script(['one\ntwo']), 'one\ntwo')
|
||||||
|
|
||||||
|
def test_format_script_newline_char(self):
|
||||||
|
self.assertEqual(metadata._format_script(['one\\ntwo']), 'one\\ntwo')
|
||||||
|
|
||||||
def test_format_stringmap_empty(self):
|
def test_format_stringmap_empty(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
metadata._format_stringmap('🔥', 'test', dict()),
|
metadata._format_stringmap('🔥', 'test', dict()),
|
||||||
@ -2019,6 +2066,89 @@ class MetadataTest(unittest.TestCase):
|
|||||||
appid, field, {afname: {'uz': 'a', locale: 'b', 'zh': 'c'}}, versionCode
|
appid, field, {afname: {'uz': 'a', locale: 'b', 'zh': 'c'}}, versionCode
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_app_to_yaml_one_category(self):
|
||||||
|
"""Categories does not get simplified to string when outputting YAML."""
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._app_to_yaml({'Categories': ['one']}),
|
||||||
|
{'Categories': ['one']},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_app_to_yaml_categories(self):
|
||||||
|
"""Sort case-insensitive before outputting YAML."""
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._app_to_yaml({'Categories': ['c', 'a', 'B']}),
|
||||||
|
{'Categories': ['a', 'B', 'c']},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_builds_to_yaml_gradle_yes(self):
|
||||||
|
app = {'Builds': [{'versionCode': 0, 'gradle': ['yes']}]}
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._builds_to_yaml(app), [{'versionCode': 0, 'gradle': ['yes']}]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_builds_to_yaml_gradle_off(self):
|
||||||
|
app = {'Builds': [{'versionCode': 0, 'gradle': ['off']}]}
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._builds_to_yaml(app), [{'versionCode': 0, 'gradle': ['off']}]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_builds_to_yaml_gradle_true(self):
|
||||||
|
app = {'Builds': [{'versionCode': 0, 'gradle': ['true']}]}
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._builds_to_yaml(app), [{'versionCode': 0, 'gradle': ['true']}]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_builds_to_yaml_gradle_false(self):
|
||||||
|
app = {'Builds': [{'versionCode': 0, 'gradle': ['false']}]}
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._builds_to_yaml(app), [{'versionCode': 0, 'gradle': ['false']}]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_builds_to_yaml_stripped(self):
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._builds_to_yaml(
|
||||||
|
{
|
||||||
|
'Builds': [
|
||||||
|
metadata.Build({'versionCode': 0, 'rm': [None], 'init': ['']})
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
[{'versionCode': 0}],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_builds_to_yaml(self):
|
||||||
|
"""Include one of each flag type with a valid value."""
|
||||||
|
app = {
|
||||||
|
'Builds': [
|
||||||
|
metadata.Build(
|
||||||
|
{
|
||||||
|
'versionCode': 0,
|
||||||
|
'gradle': ['free'],
|
||||||
|
'rm': ['0', '2'],
|
||||||
|
'submodules': True,
|
||||||
|
'timeout': 0,
|
||||||
|
'init': ['false', 'two'],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
}
|
||||||
|
# check that metadata.Build() inited flag values
|
||||||
|
self.assertEqual(app['Builds'][0]['scanignore'], list())
|
||||||
|
# then unchanged values should be removed by _builds_to_yaml
|
||||||
|
self.assertEqual(
|
||||||
|
metadata._builds_to_yaml(app),
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'versionCode': 0,
|
||||||
|
'gradle': ['free'],
|
||||||
|
'rm': ['0', '2'],
|
||||||
|
'submodules': True,
|
||||||
|
'timeout': 0,
|
||||||
|
'init': ['false', 'two'],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class PostMetadataParseTest(unittest.TestCase):
|
class PostMetadataParseTest(unittest.TestCase):
|
||||||
"""Test the functions that post process the YAML input.
|
"""Test the functions that post process the YAML input.
|
||||||
|
Loading…
Reference in New Issue
Block a user