From 0f90ab9aac76209921bf794ffc3d0a93676d3dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 1 May 2017 21:19:51 +0200 Subject: [PATCH] rewrite to yaml works for app data now (builds still missing) --- fdroidserver/metadata.py | 67 +++++++++++++++++++++- tests/rewrite-metadata/fake.ota.update.yml | 37 ++++++++++++ tests/rewritemeta.TestCase | 54 +++++++++++++++++ 3 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 tests/rewrite-metadata/fake.ota.update.yml create mode 100755 tests/rewritemeta.TestCase diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 0c9d4831..4e7af381 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -25,7 +25,7 @@ import html import logging import textwrap import io - +import ruamel.yaml import yaml # use libyaml if it is available try: @@ -35,6 +35,8 @@ except ImportError: from yaml import Loader YamlLoader = Loader +from collections import OrderedDict + import fdroidserver.common from fdroidserver.exception import MetaDataException @@ -953,6 +955,69 @@ def parse_yaml_metadata(mf, app): app.update(yamlinfo) return app +def write_yaml(mf, app): + + def _class_as_dict_representer(dumper, data): + '''Creates a YAML representation of a App/Build instance''' + return dumper.represent_dict(data) + + empty_keys = [k for k, v in app.items() if not v] + for k in empty_keys: + del app[k] + + for k in ['added', 'lastUpdated', 'id', 'metadatapath']: + if k in app: + del app[k] + + #yaml.add_representer(fdroidserver.metadata.App, _class_as_dict_representer) + #ruamel.yaml.add_representer(fdroidserver.metadata.Build, _class_as_dict_representer) + #yaml.dump(app.asOrderedDict(), mf, default_flow_style=False, Dumper=yamlordereddictloader.Dumper) + + yaml_app_field_order = [ + 'Categories', + 'License', + 'Web Site', + 'Source Code', + 'Issue Tracker', + 'Donate', + 'Bitcoin', + '\n', + 'Auto Name', + 'Summary', + 'Description', + '\n', + 'Repo Type', + 'Repo', + '\n', + 'Auto Update Mode', + 'Update Check Mode', + 'Current Version', + 'Current Version Code', + ] + + preformated = ruamel.yaml.comments.CommentedMap() + insert_newline = False + for field in yaml_app_field_order: + if field is '\n': + insert_newline = True + else: + f = field.replace(' ', '') + if hasattr(app, f) and getattr(app, f): + if f in ['Description']: + preformated.update({f: ruamel.yaml.scalarstring.preserve_literal(getattr(app, f))}) + else: + preformated.update({f: getattr(app, f)}) + if insert_newline: + insert_newline = False + # inserting empty lines is not supported so we add a + # bogus comment and over-write its value + preformated.yaml_set_comment_before_after_key(f, 'bogus') + preformated.ca.items[f][1][0].value = '\n' + # TODO implement dump for builds + del(preformated['builds']) + + ruamel.yaml.round_trip_dump(preformated, mf, indent=4, block_seq_indent=2) + def write_yaml(mf, app): diff --git a/tests/rewrite-metadata/fake.ota.update.yml b/tests/rewrite-metadata/fake.ota.update.yml new file mode 100644 index 00000000..80a46780 --- /dev/null +++ b/tests/rewrite-metadata/fake.ota.update.yml @@ -0,0 +1,37 @@ +Categories: + - System +License: Apache-2.0 +WebSite: https://f-droid.org +SourceCode: https://gitlab.com/fdroid/privileged-extension +IssueTracker: https://gitlab.com/fdroid/privileged-extension/issues +Donate: https://f-droid.org/about + +AutoName: Fake OTA Update +Summary: Tests whether OTA ZIP files are being include +Description: |- + F-Droid can make use of system privileges or permissions to + install, update and remove applications on its own. The only way to obtain those + privileges is to become a system app. + + This is where the Privileged Extension comes in - being a separate app and much + smaller, it can be installed as a system app and communicate with the main app + via AIDL IPC. + + This has several advantages: + + * Reduced disk usage in the system partition + * System updates don't remove F-Droid + * The process of installing into system via root is safer + + This is packaged as an OTA (Over-The-Air) update ZIP file. It must be installed + using TWRP or other Android recovery that can flash updates to the system from + the /data/data/org.fdroid.fdroid folder on the /data partition. The standalone + APK is called F-Droid Privileged Extension. + +RepoType: git +Repo: https://gitlab.com/fdroid/privileged-extension.git + +AutoUpdateMode: Version %v +UpdateCheckMode: Tags +CurrentVersion: 0.2.1 +CurrentVersionCode: '2000' diff --git a/tests/rewritemeta.TestCase b/tests/rewritemeta.TestCase new file mode 100755 index 00000000..ade55548 --- /dev/null +++ b/tests/rewritemeta.TestCase @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import inspect +import optparse +import os +import sys +import shutil +import unittest +import yaml +import tempfile +import textwrap + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.common +import fdroidserver.metadata + + +class RewritemetaTest(unittest.TestCase): + ''' fdroidserver/metadata.py''' + + def test_rewrite_yaml(self): + + # setup/reset test dir if necessary and setup params + tmpdir = os.path.join(os.path.dirname(__file__), '..', '.testfiles') + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + testdir = tempfile.mkdtemp(prefix='test_rewrite_metadata_', dir=tmpdir) + fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']} + + # rewrite metadata + allapps = fdroidserver.metadata.read_metadata(xref=True) + for appid, app in allapps.items(): + if appid == 'fake.ota.update': + fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app) + + # assert rewrite result + with open(os.path.join(testdir, 'fake.ota.update.yml'), 'r', encoding='utf-8') as result: + with open('rewrite-metadata/fake.ota.update.yml', 'r', encoding='utf-8') as orig: + self.maxDiff = None + self.assertEqual(result.read(), orig.read()) + +if __name__ == "__main__": + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(RewritemetaTest)) + unittest.main()