#!/usr/bin/env python3 import copy import io import logging import optparse import os import random import shutil import sys import unittest from unittest import mock import tempfile import textwrap from collections import OrderedDict from pathlib import Path from testcommon import TmpCwd from ruamel.yaml import YAML yaml = YAML(typ='safe') localmodule = Path(__file__).resolve().parent.parent print('localmodule: ' + str(localmodule)) if localmodule not in sys.path: sys.path.insert(0, str(localmodule)) import fdroidserver from fdroidserver import metadata from fdroidserver.exception import MetaDataException class MetadataTest(unittest.TestCase): '''fdroidserver/metadata.py''' def setUp(self): logging.basicConfig(level=logging.DEBUG) self.basedir = localmodule / 'tests' os.chdir(self.basedir) def tearDown(self): # auto-generated dirs by functions, not tests, so they are not always cleaned up try: os.rmdir("srclibs") except OSError: pass try: os.rmdir("tmp") except OSError: pass def test_fieldtypes_key_exist(self): for k in fdroidserver.metadata.fieldtypes.keys(): self.assertTrue(k in fdroidserver.metadata.yaml_app_fields) def test_build_flagtypes_key_exist(self): for k in fdroidserver.metadata.flagtypes.keys(): self.assertTrue(k in fdroidserver.metadata.build_flags) def test_FieldValidator_BitcoinAddress(self): validator = None for vali in fdroidserver.metadata.valuetypes: if vali.name == 'Bitcoin address': validator = vali break self.assertIsNotNone(validator, "could not find 'Bitcoin address' validator") fdroidserver.metadata.warnings_action = 'error' # some valid addresses (P2PKH, P2SH, Bech32) self.assertIsNone( validator.check('1BrrrrErsrWetrTrnrrrrm4GFg7xJaNVN2', 'fake.app.id') ) self.assertIsNone( validator.check('3JrrrrWrEZr3rNrrvrecrnyirrnqRhWNLy', 'fake.app.id') ) self.assertIsNone( validator.check('bc1qar0srrr7xrkvr5lr43lrdnwrre5rgtrzrf5rrq', 'fake.app.id') ) # some invalid addresses self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, '21BvMrSYsrWrtrrlL5A10mlGFr7rrarrN2', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, '5Hrgr3ur5rGLrfKrrrrrrHSrqJrroGrrzrQrrrrrrLNrsrDrrrA', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, '92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, 'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, 'L1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, 'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id', ) def test_FieldValidator_LitecoinAddress(self): validator = None for vali in fdroidserver.metadata.valuetypes: if vali.name == 'Litecoin address': validator = vali break self.assertIsNotNone(validator, "could not find 'Litecoin address' validator") fdroidserver.metadata.warnings_action = 'error' # some valid addresses (L, M, 3) self.assertIsNone( validator.check('LgeGrrrrJAxyXprrPrrBrrX5Qrrrrrrrrd', 'fake.app.id') ) self.assertIsNone( validator.check('MrrrrrrrJAxyXpanPtrrRAX5QHxvUJo8id', 'fake.app.id') ) self.assertIsNone(validator.check('3rereVr9rAryrranrrrrrAXrrHx', 'fake.app.id')) # some invalid addresses (various special use/testnet addresses, invalid chars) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, '21BvMrSYsrWrtrrrn5Au4l4GFr7rrarrN2', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, '5Hrgr3ur5rGLrfKrrrrrr1SrqJrroGrrzrQrrrrrrLNrsrDrrrA', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, '92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, 'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, 'L0000rSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id', ) self.assertRaises( fdroidserver.exception.MetaDataException, validator.check, 'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id', ) def test_valid_funding_yml_regex(self): """Check the regex can find all the cases""" with (self.basedir / 'funding-usernames.yaml').open() as fp: data = yaml.load(fp) for k, entries in data.items(): for entry in entries: m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry) if k == 'custom': pass elif k == 'bad': self.assertIsNone( m, 'this is an invalid %s username: {%s}' % (k, entry) ) else: self.assertIsNotNone( m, 'this is a valid %s username: {%s}' % (k, entry) ) def test_read_metadata(self): def _build_yaml_representer(dumper, data): '''Creates a YAML representation of a Build instance''' return dumper.represent_dict(data) self.maxDiff = None config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.metadata.warnings_action = None apps = fdroidserver.metadata.read_metadata() for appid in ( 'org.smssecure.smssecure', 'org.adaway', 'org.videolan.vlc', 'com.politedroid', ): savepath = Path('metadata/dump') / (appid + '.yaml') frommeta = dict(apps[appid]) self.assertTrue(appid in apps) with savepath.open('r') as f: from_yaml = yaml.load(f) self.assertEqual(frommeta, from_yaml) # comment above assert and uncomment below to update test # files when new metadata fields are added # with savepath.open('w') as f: # yaml.add_representer(fdroidserver.metadata.Build, _build_yaml_representer) # yaml.dump(frommeta, f, default_flow_style=False) def test_rewrite_yaml_fakeotaupdate(self): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) fdroidserver.common.config = {'accepted_formats': ['yml']} fdroidserver.metadata.warnings_action = None # rewrite metadata allapps = fdroidserver.metadata.read_metadata() for appid, app in allapps.items(): if appid == 'fake.ota.update': fdroidserver.metadata.write_metadata( testdir / (appid + '.yml'), app ) # assert rewrite result self.maxDiff = None file_name = 'fake.ota.update.yml' self.assertEqual( (testdir / file_name).read_text(encoding='utf-8'), (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_rewrite_yaml_fdroidclient(self): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) fdroidserver.common.config = {'accepted_formats': ['yml']} # rewrite metadata allapps = fdroidserver.metadata.read_metadata() for appid, app in allapps.items(): if appid == 'org.fdroid.fdroid': fdroidserver.metadata.write_metadata( testdir / (appid + '.yml'), app ) # assert rewrite result self.maxDiff = None file_name = 'org.fdroid.fdroid.yml' self.assertEqual( (testdir / file_name).read_text(encoding='utf-8'), (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_rewrite_yaml_special_build_params(self): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) # rewrite metadata allapps = fdroidserver.metadata.read_metadata() for appid, app in allapps.items(): if appid == 'app.with.special.build.params': fdroidserver.metadata.write_metadata( testdir / (appid + '.yml'), app ) # assert rewrite result self.maxDiff = None file_name = 'app.with.special.build.params.yml' self.assertEqual( (testdir / file_name).read_text(encoding='utf-8'), (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_post_parse_yaml_metadata(self): fdroidserver.metadata.warnings_action = 'error' yamldata = OrderedDict() builds = [] yamldata['Builds'] = builds build = OrderedDict() builds.append(build) build['versionCode'] = 1.1 self.assertRaises( fdroidserver.exception.MetaDataException, fdroidserver.metadata.post_parse_yaml_metadata, yamldata, ) build['versionCode'] = '1' self.assertRaises( fdroidserver.exception.MetaDataException, fdroidserver.metadata.post_parse_yaml_metadata, yamldata, ) build['versionCode'] = 1 build['versionName'] = 1 fdroidserver.metadata.post_parse_yaml_metadata(yamldata) self.assertNotEqual(1, yamldata['Builds'][0]['versionName']) self.assertEqual('1', yamldata['Builds'][0]['versionName']) self.assertEqual(1, yamldata['Builds'][0]['versionCode']) build['versionName'] = 1.0 fdroidserver.metadata.post_parse_yaml_metadata(yamldata) self.assertNotEqual(1.0, yamldata['Builds'][0]['versionName']) self.assertEqual('1.0', yamldata['Builds'][0]['versionName']) build['commit'] = 1.0 fdroidserver.metadata.post_parse_yaml_metadata(yamldata) self.assertNotEqual(1.0, yamldata['Builds'][0]['commit']) self.assertEqual('1.0', yamldata['Builds'][0]['commit']) teststr = '98234fab134b' build['commit'] = teststr fdroidserver.metadata.post_parse_yaml_metadata(yamldata) self.assertEqual(teststr, yamldata['Builds'][0]['commit']) testcommitid = 1234567890 build['commit'] = testcommitid fdroidserver.metadata.post_parse_yaml_metadata(yamldata) self.assertNotEqual(testcommitid, yamldata['Builds'][0]['commit']) self.assertEqual('1234567890', yamldata['Builds'][0]['commit']) def test_read_metadata_sort_by_time(self): with tempfile.TemporaryDirectory() as testdir, TmpCwd(testdir): testdir = Path(testdir) metadatadir = testdir / 'metadata' metadatadir.mkdir() randomlist = [] randomapps = list((self.basedir / 'metadata').glob('*.yml')) random.shuffle(randomapps) i = 1 for f in randomapps: shutil.copy(f, metadatadir) new = metadatadir / f.name stat = new.stat() os.utime(new, (stat.st_ctime, stat.st_mtime + i)) # prepend new item so newest is always first randomlist = [f.stem] + randomlist i += 1 allapps = fdroidserver.metadata.read_metadata(sort_by_time=True) allappids = [] for appid, app in allapps.items(): allappids.append(appid) self.assertEqual(randomlist, allappids) def test_parse_yaml_metadata_0size_file(self): mf = io.StringIO('') mf.name = 'mock_filename.yaml' self.assertEqual(fdroidserver.metadata.parse_yaml_metadata(mf), dict()) def test_parse_yaml_metadata_empty_dict_file(self): mf = io.StringIO('{}') mf.name = 'mock_filename.yaml' self.assertEqual(fdroidserver.metadata.parse_yaml_metadata(mf), dict()) def test_parse_yaml_metadata_empty_string_file(self): mf = io.StringIO('""') mf.name = 'mock_filename.yaml' self.assertEqual(fdroidserver.metadata.parse_yaml_metadata(mf), dict()) def test_parse_yaml_metadata_unknown_app_field(self): mf = io.StringIO( textwrap.dedent( """\ AutoName: F-Droid RepoType: git Builds: [] bad: value""" ) ) mf.name = 'mock_filename.yaml' with mock.patch('fdroidserver.metadata.warnings_action', 'error'): with self.assertRaises(MetaDataException): fdroidserver.metadata.parse_yaml_metadata(mf) def test_parse_yaml_metadata_unknown_build_flag(self): mf = io.StringIO( textwrap.dedent( """\ AutoName: F-Droid RepoType: git Builds: - bad: value""" ) ) mf.name = 'mock_filename.yaml' with mock.patch('fdroidserver.metadata.warnings_action', 'error'): with self.assertRaises(MetaDataException): fdroidserver.metadata.parse_yaml_metadata(mf) def test_parse_yaml_srclib_corrupt_file(self): with tempfile.TemporaryDirectory() as testdir: testdir = Path(testdir) srclibfile = testdir / 'srclib/mock.yml' srclibfile.parent.mkdir() with srclibfile.open('w') as fp: fp.write( textwrap.dedent( """ - RepoType: git - Repo: https://github.com/realm/realm-js.git """ ) ) with mock.patch('fdroidserver.metadata.warnings_action', 'error'): with self.assertRaises(MetaDataException): fdroidserver.metadata.parse_yaml_srclib(srclibfile) def test_write_yaml_with_placeholder_values(self): mf = io.StringIO() app = fdroidserver.metadata.App() app.Categories = ['None'] app.SourceCode = "https://gitlab.com/fdroid/fdroidclient.git" app.IssueTracker = "https://gitlab.com/fdroid/fdroidclient/issues" app.RepoType = 'git' app.Repo = 'https://gitlab.com/fdroid/fdroidclient.git' app.AutoUpdateMode = 'None' app.UpdateCheckMode = 'Tags' build = fdroidserver.metadata.Build() build.versionName = 'Unknown' # taken from fdroidserver/import.py build.versionCode = '0' # taken from fdroidserver/import.py build.disable = 'Generated by import.py ...' build.commit = 'Unknown' build.gradle = [True] app['Builds'] = [build] fdroidserver.metadata.write_yaml(mf, app) mf.seek(0) self.assertEqual( mf.read(), textwrap.dedent( """\ Categories: - None License: Unknown SourceCode: https://gitlab.com/fdroid/fdroidclient.git IssueTracker: https://gitlab.com/fdroid/fdroidclient/issues RepoType: git Repo: https://gitlab.com/fdroid/fdroidclient.git Builds: - versionName: Unknown versionCode: 0 disable: Generated by import.py ... commit: Unknown gradle: - true AutoUpdateMode: None UpdateCheckMode: Tags """ ), ) def test_parse_yaml_metadata_prebuild_list(self): mf = io.StringIO( textwrap.dedent( """\ AutoName: F-Droid RepoType: git Builds: - versionCode: 1 versionName: v0.1.0 sudo: - apt-get update - apt-get install -y whatever - sed -i -e 's/> /a/file build: - ./gradlew someSpecialTask - sed -i 'd/that wrong config/' gradle.properties - ./gradlew compile """ ) ) mf.name = 'mock_filename.yaml' mf.seek(0) with mock.patch('fdroidserver.metadata.warnings_action', 'error'): result = fdroidserver.metadata.parse_yaml_metadata(mf) self.maxDiff = None self.assertDictEqual( result, { 'AutoName': 'F-Droid', 'RepoType': 'git', 'Builds': [ { 'versionCode': 1, 'versionName': 'v0.1.0', 'sudo': [ "apt-get update", "apt-get install -y whatever", "sed -i -e 's/> /a/file", ], 'build': [ "./gradlew someSpecialTask", "sed -i 'd/that wrong config/' gradle.properties", "./gradlew compile", ], } ], }, ) def test_parse_yaml_metadata_prebuild_strings(self): mf = io.StringIO( textwrap.dedent( """\ AutoName: F-Droid RepoType: git Builds: - versionCode: 1 versionName: v0.1.0 sudo: |- apt-get update && apt-get install -y whatever && sed -i -e 's/> /a/file build: |- ./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile """ ) ) mf.name = 'mock_filename.yaml' mf.seek(0) with mock.patch('fdroidserver.metadata.warnings_action', 'error'): result = fdroidserver.metadata.parse_yaml_metadata(mf) self.maxDiff = None self.assertDictEqual( result, { 'AutoName': 'F-Droid', 'RepoType': 'git', 'Builds': [ { 'versionCode': 1, 'versionName': 'v0.1.0', 'sudo': [ "apt-get update && " "apt-get install -y whatever && " "sed -i -e 's/> /a/file" ], 'build': [ "./gradlew someSpecialTask && " "sed -i 'd/that wrong config/' gradle.properties && " "./gradlew compile" ], } ], }, ) def test_parse_yaml_metadata_prebuild_string(self): mf = io.StringIO( textwrap.dedent( """\ AutoName: F-Droid RepoType: git Builds: - versionCode: 1 versionName: v0.1.0 prebuild: |- a && b && sed -i 's,a,b,' """ ) ) mf.name = 'mock_filename.yaml' mf.seek(0) with mock.patch('fdroidserver.metadata.warnings_action', 'error'): result = fdroidserver.metadata.parse_yaml_metadata(mf) self.assertDictEqual( result, { 'AutoName': 'F-Droid', 'RepoType': 'git', 'Builds': [ { 'versionCode': 1, 'versionName': 'v0.1.0', 'prebuild': ["a && b && " "sed -i 's,a,b,'"], } ], }, ) def test_parse_yaml_provides_should_be_ignored(self): mf = io.StringIO( textwrap.dedent( """\ Provides: this.is.deprecated AutoName: F-Droid RepoType: git Builds: - versionCode: 1 versionName: v0.1.0 prebuild: |- a && b && sed -i 's,a,b,' """ ) ) mf.name = 'mock_filename.yaml' mf.seek(0) with mock.patch('fdroidserver.metadata.warnings_action', 'error'): result = fdroidserver.metadata.parse_yaml_metadata(mf) self.assertNotIn('Provides', result) self.assertNotIn('provides', result) def test_write_yaml_1_line_scripts_as_string(self): mf = io.StringIO() app = fdroidserver.metadata.App() app.Categories = ['None'] app['Builds'] = [] build = fdroidserver.metadata.Build() build.versionCode = 102030 build.versionName = 'v1.2.3' build.sudo = ["chmod +rwx /opt"] build.init = ["sed -i -e 'g/what/ever/' /some/file"] build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"] build.build = ["./gradlew compile"] app['Builds'].append(build) fdroidserver.metadata.write_yaml(mf, app) mf.seek(0) self.assertEqual( mf.read(), textwrap.dedent( """\ Categories: - None License: Unknown Builds: - versionName: v1.2.3 versionCode: 102030 sudo: chmod +rwx /opt init: sed -i -e 'g/what/ever/' /some/file prebuild: sed -i 'd/that wrong config/' gradle.properties build: ./gradlew compile AutoUpdateMode: None UpdateCheckMode: None """ ), ) def test_write_yaml_1_line_scripts_as_list(self): mf = io.StringIO() app = fdroidserver.metadata.App() app.Categories = ['None'] app['Builds'] = [] build = fdroidserver.metadata.Build() build.versionCode = 102030 build.versionName = 'v1.2.3' build.sudo = ["chmod +rwx /opt"] build.init = ["sed -i -e 'g/what/ever/' /some/file"] build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"] build.build = ["./gradlew compile"] app['Builds'].append(build) fdroidserver.metadata.write_yaml(mf, app) mf.seek(0) self.assertEqual( mf.read(), textwrap.dedent( """\ Categories: - None License: Unknown Builds: - versionName: v1.2.3 versionCode: 102030 sudo: chmod +rwx /opt init: sed -i -e 'g/what/ever/' /some/file prebuild: sed -i 'd/that wrong config/' gradle.properties build: ./gradlew compile AutoUpdateMode: None UpdateCheckMode: None """ ), ) def test_write_yaml_multiline_scripts_from_list(self): mf = io.StringIO() app = fdroidserver.metadata.App() app.Categories = ['None'] app['Builds'] = [] build = fdroidserver.metadata.Build() build.versionCode = 102030 build.versionName = 'v1.2.3' build.sudo = [ "apt-get update", "apt-get install -y whatever", "sed -i -e 's/> /a/file"] build.build = [ "./gradlew someSpecialTask", "sed -i 'd/that wrong config/' gradle.properties", "./gradlew compile", ] app['Builds'].append(build) fdroidserver.metadata.write_yaml(mf, app) mf.seek(0) self.assertEqual( mf.read(), textwrap.dedent( """\ Categories: - None License: Unknown Builds: - versionName: v1.2.3 versionCode: 102030 sudo: - apt-get update - apt-get install -y whatever - sed -i -e 's/> /a/file build: - ./gradlew someSpecialTask - sed -i 'd/that wrong config/' gradle.properties - ./gradlew compile AutoUpdateMode: None UpdateCheckMode: None """ ), ) def test_write_yaml_multiline_scripts_from_string(self): mf = io.StringIO() app = fdroidserver.metadata.App() app.Categories = ['None'] app['Builds'] = [] build = fdroidserver.metadata.Build() build.versionCode = 102030 build.versionName = 'v1.2.3' build.sudo = [ "apt-get update", "apt-get install -y whatever", "sed -i -e 's/> /a/file"] build.build = [ "./gradlew someSpecialTask", "sed -i 'd/that wrong config/' gradle.properties", "./gradlew compile", ] app['Builds'].append(build) fdroidserver.metadata.write_yaml(mf, app) mf.seek(0) self.assertEqual( mf.read(), textwrap.dedent( """\ Categories: - None License: Unknown Builds: - versionName: v1.2.3 versionCode: 102030 sudo: - apt-get update - apt-get install -y whatever - sed -i -e 's/> /a/file build: - ./gradlew someSpecialTask - sed -i 'd/that wrong config/' gradle.properties - ./gradlew compile AutoUpdateMode: None UpdateCheckMode: None """ ), ) def test_write_yaml_make_sure_provides_does_not_get_written(self): mf = io.StringIO() app = fdroidserver.metadata.App() app.Categories = ['None'] app.Provides = 'this.is.deprecated' app['Builds'] = [] build = fdroidserver.metadata.Build() build.versionCode = 102030 build.versionName = 'v1.2.3' build.gradle = ['yes'] app['Builds'].append(build) fdroidserver.metadata.write_yaml(mf, app) mf.seek(0) self.assertEqual( mf.read(), textwrap.dedent( """\ Categories: - None License: Unknown Builds: - versionName: v1.2.3 versionCode: 102030 gradle: - yes AutoUpdateMode: None UpdateCheckMode: None """ ), ) def test_parse_yaml_srclib_unknown_key(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with Path('test.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ RepoType: git Repo: https://example.com/test.git Evil: I should not be here. ''' ) ) with self.assertRaisesRegex( MetaDataException, "Invalid srclib metadata: " "unknown key 'Evil' in " "'test.yml'", ): fdroidserver.metadata.parse_yaml_srclib(Path('test.yml')) def test_parse_yaml_srclib_does_not_exists(self): fdroidserver.metadata.warnings_action = 'error' with self.assertRaisesRegex( MetaDataException, "Invalid scrlib metadata: " r"'non(/|\\)existent-test-srclib.yml' " "does not exist", ): fdroidserver.metadata.parse_yaml_srclib( Path('non/existent-test-srclib.yml') ) def test_parse_yaml_srclib_simple(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with Path('simple.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ # this should be simple RepoType: git Repo: https://git.host/repo.git ''' ) ) srclib = fdroidserver.metadata.parse_yaml_srclib(Path('simple.yml')) self.assertDictEqual( { 'Repo': 'https://git.host/repo.git', 'RepoType': 'git', 'Subdir': None, 'Prepare': None, }, srclib, ) def test_parse_yaml_srclib_simple_with_blanks(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with Path('simple.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ # this should be simple RepoType: git Repo: https://git.host/repo.git Subdir: Prepare: ''' ) ) srclib = fdroidserver.metadata.parse_yaml_srclib(Path('simple.yml')) self.assertDictEqual( { 'Repo': 'https://git.host/repo.git', 'RepoType': 'git', 'Subdir': [''], 'Prepare': [], }, srclib, ) def test_parse_yaml_srclib_Changelog_cketti(self): fdroidserver.metadata.warnings_action = 'error' with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): with Path('Changelog-cketti.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ RepoType: git Repo: https://github.com/cketti/ckChangeLog Subdir: library,ckChangeLog/src/main Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties" ''' ) ) srclib = fdroidserver.metadata.parse_yaml_srclib( Path('Changelog-cketti.yml') ) self.assertDictEqual( srclib, { 'Repo': 'https://github.com/cketti/ckChangeLog', 'RepoType': 'git', 'Subdir': ['library', 'ckChangeLog/src/main'], 'Prepare': [ "[ -f project.properties ] || echo 'source.dir=java' > " "ant.properties && echo -e " "'android.library=true\\ntarget=android-19' > project.properties" ], }, ) def test_read_srclibs_yml_subdir_list(self): fdroidserver.metadata.warnings_action = 'error' fdroidserver.metadata.srclibs = None with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): Path('srclibs').mkdir() with Path('srclibs/with-list.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ # this should be simple RepoType: git Repo: https://git.host/repo.git Subdir: - This is your last chance. - After this, there is no turning back. - You take the blue pill—the story ends, - you wake up in your bed - and believe whatever you want to believe. - You take the red pill—you stay in Wonderland - and I show you how deep the rabbit-hole goes. Prepare: There is a difference between knowing the path and walking the path. ''' ) ) fdroidserver.metadata.read_srclibs() self.maxDiff = None self.assertDictEqual( fdroidserver.metadata.srclibs, { 'with-list': { 'RepoType': 'git', 'Repo': 'https://git.host/repo.git', 'Subdir': [ 'This is your last chance.', 'After this, there is no turning back.', 'You take the blue pill—the story ends,', 'you wake up in your bed', 'and believe whatever you want to believe.', 'You take the red pill—you stay in Wonderland', 'and I show you how deep the rabbit-hole goes.', ], 'Prepare': [ 'There is a difference between knowing the path ' 'and walking the path.' ], } }, ) def test_read_srclibs_yml_prepare_list(self): fdroidserver.metadata.warnings_action = 'error' fdroidserver.metadata.srclibs = None with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): Path('srclibs').mkdir() with Path('srclibs/with-list.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ # this should be simple RepoType: git Repo: https://git.host/repo.git Subdir: Prepare: - Many - invalid - commands - here. ''' ) ) fdroidserver.metadata.read_srclibs() self.maxDiff = None self.assertDictEqual( fdroidserver.metadata.srclibs, { 'with-list': { 'RepoType': 'git', 'Repo': 'https://git.host/repo.git', 'Subdir': [''], 'Prepare': [ 'Many', 'invalid', 'commands', 'here.', ], } }, ) def test_read_srclibs(self): fdroidserver.metadata.warnings_action = 'error' fdroidserver.metadata.srclibs = None with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): Path('srclibs').mkdir() with Path('srclibs/simple.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ RepoType: git Repo: https://git.host/repo.git ''' ) ) with Path('srclibs/simple-wb.yml').open('w', encoding='utf-8') as f: f.write( textwrap.dedent( '''\ # this should be simple RepoType: git Repo: https://git.host/repo.git Subdir: Prepare: ''' ) ) fdroidserver.metadata.read_srclibs() self.assertDictEqual( fdroidserver.metadata.srclibs, { 'simple-wb': { 'RepoType': 'git', 'Repo': 'https://git.host/repo.git', 'Subdir': [''], 'Prepare': [], }, 'simple': { 'RepoType': 'git', 'Repo': 'https://git.host/repo.git', 'Subdir': None, 'Prepare': None, }, }, ) def test_build_ndk_path(self): """""" with tempfile.TemporaryDirectory(prefix='android-sdk-') as sdk_path: config = {'ndk_paths': {}, 'sdk_path': sdk_path} fdroidserver.common.config = config build = fdroidserver.metadata.Build() build.ndk = 'r10e' self.assertEqual('', build.ndk_path()) correct = '/fake/path/ndk/r21b' config['ndk_paths'] = {'r21b': correct} self.assertEqual('', build.ndk_path()) config['ndk_paths'] = {'r10e': correct} self.assertEqual(correct, build.ndk_path()) r10e = '/fake/path/ndk/r10e' r22b = '/fake/path/ndk/r22e' config['ndk_paths'] = {'r10e': r10e, 'r22b': r22b} self.assertEqual(r10e, build.ndk_path()) build.ndk = ['r10e', 'r22b'] self.assertEqual(r10e, build.ndk_path()) build.ndk = ['r22b', 'r10e'] self.assertEqual(r22b, build.ndk_path()) def test_build_ndk_path_only_accepts_str(self): """Paths in the config must be strings, never pathlib.Path instances""" config = {'ndk_paths': {'r24': Path('r24')}} fdroidserver.common.config = config build = fdroidserver.metadata.Build() build.ndk = 'r24' with self.assertRaises(TypeError): build.ndk_path() class PostMetadataParseTest(unittest.TestCase): """Test the functions that post process the YAML input. The following series of "post_metadata_parse" tests map out the current state of automatic type conversion in the YAML post processing. They are not necessary a statement of how things should be, but more to surface the details of it functions. """ def setUp(self): fdroidserver.metadata.warnings_action = 'error' def _post_metadata_parse_app_list(self, from_yaml, expected): app = {'AntiFeatures': from_yaml} metadata.post_parse_yaml_metadata(app) metadata.post_metadata_parse(app) del app['Builds'] return {'AntiFeatures': expected}, app def _post_metadata_parse_app_string(self, from_yaml, expected): app = {'Repo': from_yaml} metadata.post_parse_yaml_metadata(app) metadata.post_metadata_parse(app) del app['Builds'] return {'Repo': expected}, app def _post_metadata_parse_build_bool(self, from_yaml, expected): tested_key = 'submodules' app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]} post = copy.deepcopy(app) metadata.post_parse_yaml_metadata(post) metadata.post_metadata_parse(post) del app['Builds'][0]['versionCode'] del post['Builds'][0]['versionCode'] for build in post['Builds']: for k in list(build): if k != tested_key: del build[k] app['Builds'][0][tested_key] = expected return app, post def _post_metadata_parse_build_int(self, from_yaml, expected): tested_key = 'versionCode' app = {'Builds': [{'versionCode': from_yaml}]} post = copy.deepcopy(app) metadata.post_parse_yaml_metadata(post) metadata.post_metadata_parse(post) for build in post['Builds']: for k in list(build): if k != tested_key: del build[k] app['Builds'][0][tested_key] = expected return app, post def _post_metadata_parse_build_list(self, from_yaml, expected): tested_key = 'rm' app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]} post = copy.deepcopy(app) metadata.post_parse_yaml_metadata(post) metadata.post_metadata_parse(post) del app['Builds'][0]['versionCode'] del post['Builds'][0]['versionCode'] for build in post['Builds']: for k in list(build): if k != tested_key: del build[k] app['Builds'][0][tested_key] = expected return app, post def _post_metadata_parse_build_script(self, from_yaml, expected): tested_key = 'build' app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]} post = copy.deepcopy(app) metadata.post_parse_yaml_metadata(post) metadata.post_metadata_parse(post) del app['Builds'][0]['versionCode'] del post['Builds'][0]['versionCode'] for build in post['Builds']: for k in list(build): if k != tested_key: del build[k] app['Builds'][0][tested_key] = expected return app, post def _post_metadata_parse_build_string(self, from_yaml, expected): tested_key = 'commit' app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]} post = copy.deepcopy(app) metadata.post_parse_yaml_metadata(post) metadata.post_metadata_parse(post) del app['Builds'][0]['versionCode'] del post['Builds'][0]['versionCode'] for build in post['Builds']: for k in list(build): if k != tested_key: del build[k] app['Builds'][0][tested_key] = expected return app, post def test_post_metadata_parse_int(self): """Run the int 123456 through the various field and flag types.""" with self.assertRaises(TypeError): self._post_metadata_parse_app_list(123456, TypeError) self.assertEqual(*self._post_metadata_parse_app_string(123456, '123456')) self.assertEqual(*self._post_metadata_parse_build_bool(123456, 123456)) self.assertEqual(*self._post_metadata_parse_build_int(123456, 123456)) self.assertEqual(*self._post_metadata_parse_build_list(123456, 123456)) self.assertEqual(*self._post_metadata_parse_build_script(123456, 123456)) self.assertEqual(*self._post_metadata_parse_build_string(123456, '123456')) def test_post_metadata_parse_int_0(self): """Run the int 0 through the various field and flag types.""" self.assertEqual(*self._post_metadata_parse_app_list(0, 0)) self.assertEqual(*self._post_metadata_parse_app_string(0, 0)) self.assertEqual(*self._post_metadata_parse_build_bool(0, 0)) self.assertEqual(*self._post_metadata_parse_build_int(0, 0)) self.assertEqual(*self._post_metadata_parse_build_list(0, 0)) self.assertEqual(*self._post_metadata_parse_build_script(0, 0)) self.assertEqual(*self._post_metadata_parse_build_string(0, '0')) def test_post_metadata_parse_float_0_0(self): """Run the float 0.0 through the various field and flag types.""" self.assertEqual(*self._post_metadata_parse_app_list(0.0, 0.0)) self.assertEqual(*self._post_metadata_parse_app_string(0.0, 0.0)) self.assertEqual(*self._post_metadata_parse_build_bool(0.0, 0.0)) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int(0.0, MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list(0.0, 0.0)) self.assertEqual(*self._post_metadata_parse_build_script(0.0, 0.0)) self.assertEqual(*self._post_metadata_parse_build_string(0.0, '0.0')) def test_post_metadata_parse_float_0_1(self): """Run the float 0.1 through the various field and flag types.""" with self.assertRaises(TypeError): self._post_metadata_parse_app_list(0.1, TypeError) self.assertEqual(*self._post_metadata_parse_app_string(0.1, '0.1')) self.assertEqual(*self._post_metadata_parse_build_bool(0.1, 0.1)) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int(0.1, MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list(0.1, 0.1)) self.assertEqual(*self._post_metadata_parse_build_script(0.1, 0.1)) self.assertEqual(*self._post_metadata_parse_build_string(0.1, '0.1')) def test_post_metadata_parse_float_1_0(self): """Run the float 1.0 through the various field and flag types.""" with self.assertRaises(TypeError): self._post_metadata_parse_app_list(1.0, TypeError) self.assertEqual(*self._post_metadata_parse_app_string(1.0, '1.0')) self.assertEqual(*self._post_metadata_parse_build_bool(1.0, 1.0)) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int(1.0, MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list(1.0, 1.0)) self.assertEqual(*self._post_metadata_parse_build_script(1.0, 1.0)) self.assertEqual(*self._post_metadata_parse_build_string(1.0, '1.0')) def test_post_metadata_parse_empty_list(self): self.assertEqual(*self._post_metadata_parse_app_list(list(), list())) self.assertEqual(*self._post_metadata_parse_app_string(list(), list())) self.assertEqual(*self._post_metadata_parse_build_bool(list(), list())) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int(list(), MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list(list(), list())) self.assertEqual(*self._post_metadata_parse_build_script(list(), list())) self.assertEqual(*self._post_metadata_parse_build_string(list(), '[]')) def test_post_metadata_parse_set_of_1(self): self.assertEqual(*self._post_metadata_parse_app_list({1}, ['1'])) self.assertEqual(*self._post_metadata_parse_app_string({1}, '{1}')) self.assertEqual(*self._post_metadata_parse_build_bool({1}, {1})) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int({1}, MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list({1}, {1})) self.assertEqual(*self._post_metadata_parse_build_script({1}, {1})) self.assertEqual(*self._post_metadata_parse_build_string({1}, '{1}')) def test_post_metadata_parse_empty_dict(self): self.assertEqual(*self._post_metadata_parse_app_list(dict(), dict())) self.assertEqual(*self._post_metadata_parse_app_string(dict(), dict())) self.assertEqual(*self._post_metadata_parse_build_bool(dict(), dict())) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int(dict(), MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list(dict(), dict())) self.assertEqual(*self._post_metadata_parse_build_script(dict(), dict())) self.assertEqual(*self._post_metadata_parse_build_string(dict(), '{}')) def test_post_metadata_parse_list_int_string(self): self.assertEqual(*self._post_metadata_parse_app_list([1, 'a'], ['1', 'a'])) self.assertEqual(*self._post_metadata_parse_app_string([1, 'a'], "[1, 'a']")) self.assertEqual(*self._post_metadata_parse_build_bool([1, 'a'], [1, 'a'])) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int([1, 'a'], MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list([1, 'a'], [1, 'a'])) self.assertEqual(*self._post_metadata_parse_build_script([1, 'a'], [1, 'a'])) self.assertEqual(*self._post_metadata_parse_build_string([1, 'a'], "[1, 'a']")) def test_post_metadata_parse_dict_int_string(self): self.assertEqual(*self._post_metadata_parse_app_list({'k': 1}, ['k'])) self.assertEqual(*self._post_metadata_parse_app_string({'k': 1}, "{'k': 1}")) self.assertEqual(*self._post_metadata_parse_build_bool({'k': 1}, {'k': 1})) with self.assertRaises(MetaDataException): self._post_metadata_parse_build_int({'k': 1}, MetaDataException) self.assertEqual(*self._post_metadata_parse_build_list({'k': 1}, {'k': 1})) self.assertEqual(*self._post_metadata_parse_build_script({'k': 1}, {'k': 1})) self.assertEqual(*self._post_metadata_parse_build_string({'k': 1}, "{'k': 1}")) def test_post_metadata_parse_false(self): self.assertEqual(*self._post_metadata_parse_app_list(False, False)) self.assertEqual(*self._post_metadata_parse_app_string(False, False)) self.assertEqual(*self._post_metadata_parse_build_bool(False, False)) self.assertEqual(*self._post_metadata_parse_build_int(False, False)) self.assertEqual(*self._post_metadata_parse_build_list(False, False)) self.assertEqual(*self._post_metadata_parse_build_script(False, False)) self.assertEqual(*self._post_metadata_parse_build_string(False, 'False')) def test_post_metadata_parse_true(self): with self.assertRaises(TypeError): self._post_metadata_parse_app_list(True, TypeError) self.assertEqual(*self._post_metadata_parse_app_string(True, 'true')) self.assertEqual(*self._post_metadata_parse_build_bool(True, True)) self.assertEqual(*self._post_metadata_parse_build_int(True, True)) self.assertEqual(*self._post_metadata_parse_build_list(True, True)) self.assertEqual(*self._post_metadata_parse_build_script(True, True)) self.assertEqual(*self._post_metadata_parse_build_string(True, 'True')) if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option( "-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal", ) (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(MetadataTest)) unittest.main(failfast=False)