From 942de28fa5096c2316fd515c8ccd093d9575b806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 28 Aug 2018 10:16:13 +0200 Subject: [PATCH 1/7] yaml metadata: split prebuild build field to list --- fdroidserver/metadata.py | 6 +++++ tests/metadata.TestCase | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 832be3e7..c24bb2fe 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -1091,6 +1091,10 @@ def parse_yaml_metadata(mf, app): _("Unrecognised build flag '{build_flag}' " "in '{path}'").format(build_flag=build_flag, path=mf.name)) + + if 'prebuild' in build and type(build['prebuild']) == list: + build['prebuild'] = ' && '.join(build['prebuild']) + app.update(yamldata) return app @@ -1194,6 +1198,8 @@ def write_yaml(mf, app): continue elif value == 'yes': value = 'yes' + if field == 'prebuild': + value = value.split(' && ') b.update({field: _field_to_yaml(flagtype(field), value)}) builds.append(b) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 6b3c66ef..a9be7ec0 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -209,6 +209,58 @@ class MetadataTest(unittest.TestCase): UpdateCheckMode: Tags """)) + def test_parse_yaml_metadata_prebuild(self): + mf = io.StringIO(textwrap.dedent("""\ + AutoName: F-Droid + RepoType: git + Builds: + - versionCode: 1 + versionName: v0.1.0 + prebuild: + - a + - b + - c + """)) + mf.name = 'mock_filename.yaml' + mf.seek(0) + result = {} + with mock.patch('fdroidserver.metadata.warnings_action', 'error'): + fdroidserver.metadata.parse_yaml_metadata(mf, result) + self.assertDictEqual(result, {'AutoName': 'F-Droid', + 'RepoType': 'git', + 'Builds': [{'versionCode': 1, + 'versionName': 'v0.1.0', + 'prebuild': 'a && b && c'}]}) + + def test_write_yaml_prebuild(self): + mf = io.StringIO() + app = fdroidserver.metadata.App() + app.builds = [] + build = fdroidserver.metadata.Build() + build.versionCode = 102030 + build.versionName = 'v1.2.3' + build.prebuild = 'a && b % c && d `echo \'$e\'` && f' + app.builds.append(build) + fdroidserver.metadata.write_yaml(mf, app) + mf.seek(0) + self.assertEqual(mf.read(), textwrap.dedent("""\ + Categories: + - None + License: Unknown + + Builds: + - versionName: v1.2.3 + versionCode: 102030 + prebuild: + - a + - b % c + - d `echo '$e'` + - f + + AutoUpdateMode: None + UpdateCheckMode: None + """)) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From d0a129c216d5814888d871c0da06782a2be26e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 28 Aug 2018 10:04:05 +0200 Subject: [PATCH 2/7] add test for parsing build field prebuild as string --- tests/metadata.TestCase | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index a9be7ec0..87fd319a 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -209,7 +209,7 @@ class MetadataTest(unittest.TestCase): UpdateCheckMode: Tags """)) - def test_parse_yaml_metadata_prebuild(self): + def test_parse_yaml_metadata_prebuild_list(self): mf = io.StringIO(textwrap.dedent("""\ AutoName: F-Droid RepoType: git @@ -232,6 +232,28 @@ class MetadataTest(unittest.TestCase): 'versionName': 'v0.1.0', 'prebuild': 'a && b && c'}]}) + def test_parse_yaml_metadata_prebuild_string(self): + mf = io.StringIO(textwrap.dedent("""\ + AutoName: F-Droid + RepoType: git + Builds: + - versionCode: 1 + versionName: v0.1.0 + prebuild: |- + a && b && sed -i 's,a,b,' + """)) + mf.name = 'mock_filename.yaml' + mf.seek(0) + result = {} + with mock.patch('fdroidserver.metadata.warnings_action', 'error'): + fdroidserver.metadata.parse_yaml_metadata(mf, result) + self.assertDictEqual(result, {'AutoName': 'F-Droid', + 'RepoType': 'git', + 'Builds': [{'versionCode': 1, + 'versionName': 'v0.1.0', + 'prebuild': "a && b && " + "sed -i 's,a,b,'"}]}) + def test_write_yaml_prebuild(self): mf = io.StringIO() app = fdroidserver.metadata.App() From 723815a25bd6c66f227c403c9604e14db16a5cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 28 Aug 2018 10:06:15 +0200 Subject: [PATCH 3/7] fix metadata.Testcase:test_rewrite_yaml_special_build_params --- .../app.with.special.build.params.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/metadata-rewrite-yml/app.with.special.build.params.yml b/tests/metadata-rewrite-yml/app.with.special.build.params.yml index a8925657..e7755881 100644 --- a/tests/metadata-rewrite-yml/app.with.special.build.params.yml +++ b/tests/metadata-rewrite-yml/app.with.special.build.params.yml @@ -38,8 +38,9 @@ Builds: - FacebookSDK@sdk-version-3.0.1 rm: - libs/appbrain-sdk-android.jar - prebuild: |- - sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties && sed -i 's/Class\[\]/Class\\[\]/g' $$FacebookSDK$$/src/com/facebook/model/GraphObject.java + prebuild: + - sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties + - sed -i 's/Class\[\]/Class\\[\]/g' $$FacebookSDK$$/src/com/facebook/model/GraphObject.java - versionName: 1.2.2 versionCode: 42 @@ -52,8 +53,10 @@ Builds: - libs/appbrain-sdk-android.jar extlibs: - android/android-support-v4.jar - prebuild: |- - mv libs/android-support-v4.jar $$FacebookSDK$$/libs/ && sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties && sed -i 's/Class\[\]/Class\\[\]/g' $$FacebookSDK$$/src/com/facebook/model/GraphObject.java + prebuild: + - mv libs/android-support-v4.jar $$FacebookSDK$$/libs/ + - sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties + - sed -i 's/Class\[\]/Class\\[\]/g' $$FacebookSDK$$/src/com/facebook/model/GraphObject.java - versionName: 2.1.1 versionCode: 48 From afdc0be95445f3d834b401629abfa6db31c243d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 28 Aug 2018 10:27:56 +0200 Subject: [PATCH 4/7] yaml build flag prebuild: keep writing string rathern than list of strings when theres just 1 entry --- fdroidserver/metadata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index c24bb2fe..3a1ed4d7 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -1199,7 +1199,10 @@ def write_yaml(mf, app): elif value == 'yes': value = 'yes' if field == 'prebuild': - value = value.split(' && ') + prebuild_tokens = value.split(' && ') + # when theres just 1 entry keep string rather than a list + if len(prebuild_tokens) > 1: + value = prebuild_tokens b.update({field: _field_to_yaml(flagtype(field), value)}) builds.append(b) From a21635ae2e923685294653e3ce8198f418a349f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 14 Nov 2018 14:20:33 +0100 Subject: [PATCH 5/7] fix metadata test case: write yaml prebuild --- tests/metadata.TestCase | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 87fd319a..441fc94e 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -257,6 +257,7 @@ class MetadataTest(unittest.TestCase): def test_write_yaml_prebuild(self): mf = io.StringIO() app = fdroidserver.metadata.App() + app.Categories = ['None'] app.builds = [] build = fdroidserver.metadata.Build() build.versionCode = 102030 From c15a7508e7b6a97e7d2895d1cf548d147e66f88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 13 Nov 2018 19:53:15 +0100 Subject: [PATCH 6/7] write yaml script metadata as lists --- fdroidserver/metadata.py | 22 ++++--- tests/metadata.TestCase | 136 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 142 insertions(+), 16 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 3a1ed4d7..f85ede04 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -1150,10 +1150,17 @@ def write_yaml(mf, app): else: return str(value) elif typ is TYPE_SCRIPT: - if len(value) > 50: - return ruamel.yaml.scalarstring.preserve_literal(value) + if type(value) == list: + if len(value) == 1: + return value[0] + else: + return value else: - return value + script_lines = value.split(' && ') + if len(script_lines) > 1: + return script_lines + else: + return value else: return value @@ -1189,8 +1196,8 @@ def write_yaml(mf, app): for build in app.builds: b = ruamel.yaml.comments.CommentedMap() for field in build_flags: - if hasattr(build, field) and getattr(build, field): - value = getattr(build, field) + value = getattr(build, field) + if hasattr(build, field) and value: if field == 'gradle' and value == ['off']: value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')] if field in ('maven', 'buildozer'): @@ -1198,11 +1205,6 @@ def write_yaml(mf, app): continue elif value == 'yes': value = 'yes' - if field == 'prebuild': - prebuild_tokens = value.split(' && ') - # when theres just 1 entry keep string rather than a list - if len(prebuild_tokens) > 1: - value = prebuild_tokens b.update({field: _field_to_yaml(flagtype(field), value)}) builds.append(b) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 441fc94e..bedceb24 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -254,7 +254,7 @@ class MetadataTest(unittest.TestCase): 'prebuild': "a && b && " "sed -i 's,a,b,'"}]}) - def test_write_yaml_prebuild(self): + def test_write_yaml_1_line_scripts_as_string(self): mf = io.StringIO() app = fdroidserver.metadata.App() app.Categories = ['None'] @@ -262,7 +262,10 @@ class MetadataTest(unittest.TestCase): build = fdroidserver.metadata.Build() build.versionCode = 102030 build.versionName = 'v1.2.3' - build.prebuild = 'a && b % c && d `echo \'$e\'` && f' + build.sudo = "chmod +rwx /opt" + build.init = "sed -i -e 'g/what/ever/' /some/file" + build.prebuild = "sed -i 'd/that wrong config/' gradle.properties" + build.build = "./gradlew compile" app.builds.append(build) fdroidserver.metadata.write_yaml(mf, app) mf.seek(0) @@ -274,11 +277,132 @@ class MetadataTest(unittest.TestCase): Builds: - versionName: v1.2.3 versionCode: 102030 + sudo: chmod +rwx /opt + init: sed -i -e 'g/what/ever/' /some/file + prebuild: sed -i 'd/that wrong config/' gradle.properties + build: ./gradlew compile + + AutoUpdateMode: None + UpdateCheckMode: None + """)) + + def test_write_yaml_1_line_scripts_as_list(self): + mf = io.StringIO() + app = fdroidserver.metadata.App() + app.Categories = ['None'] + app.builds = [] + build = fdroidserver.metadata.Build() + build.versionCode = 102030 + build.versionName = 'v1.2.3' + build.sudo = ["chmod +rwx /opt"] + build.init = ["sed -i -e 'g/what/ever/' /some/file"] + build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"] + build.build = ["./gradlew compile"] + app.builds.append(build) + fdroidserver.metadata.write_yaml(mf, app) + mf.seek(0) + self.assertEqual(mf.read(), textwrap.dedent("""\ + Categories: + - None + License: Unknown + + Builds: + - versionName: v1.2.3 + versionCode: 102030 + sudo: chmod +rwx /opt + init: sed -i -e 'g/what/ever/' /some/file + prebuild: sed -i 'd/that wrong config/' gradle.properties + build: ./gradlew compile + + AutoUpdateMode: None + UpdateCheckMode: None + """)) + + def test_write_yaml_multiline_scripts_from_list(self): + mf = io.StringIO() + app = fdroidserver.metadata.App() + app.Categories = ['None'] + app.builds = [] + build = fdroidserver.metadata.Build() + build.versionCode = 102030 + build.versionName = 'v1.2.3' + build.sudo = ["apt-get update", + "apt-get install -y whatever", + "sed -i -e 's/> /a/file"] + build.build = ["./gradlew someSpecialTask", + "sed -i 'd/that wrong config/' gradle.properties", + "./gradlew compile"] + app.builds.append(build) + fdroidserver.metadata.write_yaml(mf, app) + mf.seek(0) + self.assertEqual(mf.read(), textwrap.dedent("""\ + Categories: + - None + License: Unknown + + Builds: + - versionName: v1.2.3 + versionCode: 102030 + sudo: + - apt-get update + - apt-get install -y whatever + - sed -i -e 's/> /a/file + build: + - ./gradlew someSpecialTask + - sed -i 'd/that wrong config/' gradle.properties + - ./gradlew compile + + AutoUpdateMode: None + UpdateCheckMode: None + """)) + + def test_write_yaml_multiline_scripts_from_string(self): + mf = io.StringIO() + app = fdroidserver.metadata.App() + app.Categories = ['None'] + app.builds = [] + build = fdroidserver.metadata.Build() + build.versionCode = 102030 + build.versionName = 'v1.2.3' + build.sudo = "apt-get update && apt-get install -y whatever && sed -i -e 's/> /a/file + build: + - ./gradlew someSpecialTask + - sed -i 'd/that wrong config/' gradle.properties + - ./gradlew compile AutoUpdateMode: None UpdateCheckMode: None From 31ca2092a19adc3d4b5f302dafb72a4babb882ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 14 Nov 2018 13:42:05 +0100 Subject: [PATCH 7/7] yaml parsing: script build flags can now be lists --- fdroidserver/metadata.py | 17 ++++++++--- tests/metadata.TestCase | 61 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index f85ede04..49c34678 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -1091,14 +1091,23 @@ def parse_yaml_metadata(mf, app): _("Unrecognised build flag '{build_flag}' " "in '{path}'").format(build_flag=build_flag, path=mf.name)) - - if 'prebuild' in build and type(build['prebuild']) == list: - build['prebuild'] = ' && '.join(build['prebuild']) - + post_parse_yaml_metadata(yamldata) app.update(yamldata) return app +def post_parse_yaml_metadata(yamldata): + """transform yaml metadata to our internal data format""" + for build in yamldata.get('Builds', []): + for flag in build.keys(): + _flagtype = flagtype(flag) + + # concatenate script flags into a single string if they are stored as list + if _flagtype is TYPE_SCRIPT: + if isinstance(build[flag], list): + build[flag] = ' && '.join(build[flag]) + + def write_yaml(mf, app): """Write metadata in yaml format. diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index bedceb24..486dca09 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -216,21 +216,74 @@ class MetadataTest(unittest.TestCase): Builds: - versionCode: 1 versionName: v0.1.0 + sudo: + - apt-get update + - apt-get install -y whatever + - sed -i -e 's/> /a/file + build: + - ./gradlew someSpecialTask + - sed -i 'd/that wrong config/' gradle.properties + - ./gradlew compile """)) mf.name = 'mock_filename.yaml' mf.seek(0) result = {} with mock.patch('fdroidserver.metadata.warnings_action', 'error'): fdroidserver.metadata.parse_yaml_metadata(mf, result) + self.maxDiff = None self.assertDictEqual(result, {'AutoName': 'F-Droid', 'RepoType': 'git', 'Builds': [{'versionCode': 1, 'versionName': 'v0.1.0', - 'prebuild': 'a && b && c'}]}) + 'sudo': "apt-get update && " + "apt-get install -y whatever && " + "sed -i -e 's/> /a/file", + 'build': "./gradlew someSpecialTask && " + "sed -i 'd/that wrong config/' gradle.properties && " + "./gradlew compile"}]}) + + def test_parse_yaml_metadata_prebuild_strings(self): + mf = io.StringIO(textwrap.dedent("""\ + AutoName: F-Droid + RepoType: git + Builds: + - versionCode: 1 + versionName: v0.1.0 + sudo: |- + apt-get update && apt-get install -y whatever && sed -i -e 's/> /a/file + build: |- + ./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile + """)) + mf.name = 'mock_filename.yaml' + mf.seek(0) + result = {} + with mock.patch('fdroidserver.metadata.warnings_action', 'error'): + fdroidserver.metadata.parse_yaml_metadata(mf, result) + self.maxDiff = None + self.assertDictEqual(result, {'AutoName': 'F-Droid', + 'RepoType': 'git', + 'Builds': [{'versionCode': 1, + 'versionName': 'v0.1.0', + 'sudo': "apt-get update && " + "apt-get install -y whatever && " + "sed -i -e 's/> /a/file", + 'build': "./gradlew someSpecialTask && " + "sed -i 'd/that wrong config/' gradle.properties && " + "./gradlew compile"}]}) def test_parse_yaml_metadata_prebuild_string(self): mf = io.StringIO(textwrap.dedent("""\