fdroidserver/tests/metadata.TestCase

1480 lines
56 KiB
Python
Executable File

#!/usr/bin/env python3
import copy
import io
import logging
import optparse
import os
import random
import ruamel.yaml
import shutil
import sys
import unittest
import tempfile
import textwrap
from collections import OrderedDict
from pathlib import Path
from unittest import mock
from testcommon import TmpCwd, mkdtemp
localmodule = Path(__file__).resolve().parent.parent
print('localmodule: ' + str(localmodule))
if localmodule not in sys.path:
sys.path.insert(0, str(localmodule))
import fdroidserver
from fdroidserver import metadata
from fdroidserver.exception import MetaDataException
def _get_mock_mf(s):
mf = io.StringIO(s)
mf.name = 'mock_filename.yaml'
return mf
class MetadataTest(unittest.TestCase):
'''fdroidserver/metadata.py'''
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = localmodule / 'tests'
os.chdir(self.basedir)
self._td = mkdtemp()
self.testdir = self._td.name
fdroidserver.metadata.warnings_action = 'error'
def tearDown(self):
# auto-generated dirs by functions, not tests, so they are not always cleaned up
self._td.cleanup()
try:
os.rmdir("srclibs")
except OSError:
pass
try:
os.rmdir("tmp")
except OSError:
pass
def test_fieldtypes_key_exist(self):
for k in fdroidserver.metadata.fieldtypes.keys():
self.assertTrue(k in fdroidserver.metadata.yaml_app_fields)
def test_build_flagtypes_key_exist(self):
for k in fdroidserver.metadata.flagtypes.keys():
self.assertTrue(k in fdroidserver.metadata.build_flags)
def test_FieldValidator_BitcoinAddress(self):
validator = None
for vali in fdroidserver.metadata.valuetypes:
if vali.name == 'Bitcoin address':
validator = vali
break
self.assertIsNotNone(validator, "could not find 'Bitcoin address' validator")
# some valid addresses (P2PKH, P2SH, Bech32)
self.assertIsNone(
validator.check('1BrrrrErsrWetrTrnrrrrm4GFg7xJaNVN2', 'fake.app.id')
)
self.assertIsNone(
validator.check('3JrrrrWrEZr3rNrrvrecrnyirrnqRhWNLy', 'fake.app.id')
)
self.assertIsNone(
validator.check('bc1qar0srrr7xrkvr5lr43lrdnwrre5rgtrzrf5rrq', 'fake.app.id')
)
# some invalid addresses
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'21BvMrSYsrWrtrrlL5A10mlGFr7rrarrN2',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'5Hrgr3ur5rGLrfKrrrrrrHSrqJrroGrrzrQrrrrrrLNrsrDrrrA',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'L1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx',
'fake.app.id',
)
def test_FieldValidator_LitecoinAddress(self):
validator = None
for vali in fdroidserver.metadata.valuetypes:
if vali.name == 'Litecoin address':
validator = vali
break
self.assertIsNotNone(validator, "could not find 'Litecoin address' validator")
# some valid addresses (L, M, 3, segwit)
self.assertIsNone(
validator.check('LgeGrrrrJAxyXprrPrrBrrX5Qrrrrrrrrd', 'fake.app.id')
)
self.assertIsNone(
validator.check('MrrrrrrrJAxyXpanPtrrRAX5QHxvUJo8id', 'fake.app.id')
)
self.assertIsNone(validator.check('3rereVr9rAryrranrrrrrAXrrHx', 'fake.app.id'))
self.assertIsNone(
validator.check(
'ltc1q7euacwhn6ef99vcfa57mute92q572aqsc4c2j5', 'fake.app.id'
)
)
# some invalid addresses (various special use/testnet addresses, invalid chars)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'21BvMrSYsrWrtrrrn5Au4l4GFr7rrarrN2',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'5Hrgr3ur5rGLrfKrrrrrr1SrqJrroGrrzrQrrrrrrLNrsrDrrrA',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'L0000rSYsrWrtrrrn5Au4m4GFr7rrarrN2',
'fake.app.id',
)
self.assertRaises(
fdroidserver.exception.MetaDataException,
validator.check,
'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx',
'fake.app.id',
)
def test_valid_funding_yml_regex(self):
"""Check the regex can find all the cases"""
with (self.basedir / 'funding-usernames.yaml').open() as fp:
yaml = ruamel.yaml.YAML(typ='safe')
data = yaml.load(fp)
for k, entries in data.items():
for entry in entries:
m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry)
if k == 'custom':
pass
elif k == 'bad':
self.assertIsNone(
m, 'this is an invalid %s username: {%s}' % (k, entry)
)
else:
self.assertIsNotNone(
m, 'this is a valid %s username: {%s}' % (k, entry)
)
@mock.patch('git.Repo')
def test_read_metadata(self, git_repo):
"""Read specified metadata files included in tests/, compare to stored output"""
self.maxDiff = None
config = dict()
fdroidserver.common.fill_config_defaults(config)
fdroidserver.common.config = config
fdroidserver.metadata.warnings_action = None
yaml = ruamel.yaml.YAML(typ='safe')
apps = fdroidserver.metadata.read_metadata()
for appid in (
'org.smssecure.smssecure',
'org.adaway',
'org.videolan.vlc',
'com.politedroid',
):
savepath = Path('metadata/dump') / (appid + '.yaml')
frommeta = dict(apps[appid])
self.assertTrue(appid in apps)
with savepath.open('r') as f:
from_yaml = yaml.load(f)
self.assertEqual(frommeta, from_yaml)
# comment above assert and uncomment below to update test
# files when new metadata fields are added
# with savepath.open('w') as fp:
# yaml.default_flow_style = False
# yaml.register_class(metadata.Build)
# yaml.dump(frommeta, fp)
@mock.patch('git.Repo')
def test_metadata_overrides_dot_fdroid_yml(self, git_Repo):
"""Fields in metadata files should override anything in .fdroid.yml."""
app = metadata.parse_metadata('metadata/info.guardianproject.urzip.yml')
self.assertEqual(app['Summary'], '一个实用工具,获取已安装在您的设备上的应用的有关信息')
def test_dot_fdroid_yml_works_without_git(self):
"""Parsing should work if .fdroid.yml is present and it is not a git repo."""
os.chdir(self.testdir)
yml = Path('metadata/test.yml')
yml.parent.mkdir()
with yml.open('w') as fp:
fp.write('Repo: https://example.com/not/git/or/anything')
fdroid_yml = Path('build/test/.fdroid.yml')
fdroid_yml.parent.mkdir(parents=True)
with fdroid_yml.open('w') as fp:
fp.write('OpenCollective: test')
metadata.parse_metadata(yml) # should not throw an exception
@mock.patch('git.Repo')
def test_rewrite_yaml_fakeotaupdate(self, git_Repo):
with tempfile.TemporaryDirectory() as testdir:
testdir = Path(testdir)
fdroidserver.common.config = {'accepted_formats': ['yml']}
fdroidserver.metadata.warnings_action = None
# rewrite metadata
allapps = fdroidserver.metadata.read_metadata()
for appid, app in allapps.items():
if appid == 'fake.ota.update':
fdroidserver.metadata.write_metadata(
testdir / (appid + '.yml'), app
)
# assert rewrite result
self.maxDiff = None
file_name = 'fake.ota.update.yml'
self.assertEqual(
(testdir / file_name).read_text(encoding='utf-8'),
(Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
)
@mock.patch('git.Repo')
def test_rewrite_yaml_fdroidclient(self, git_Repo):
with tempfile.TemporaryDirectory() as testdir:
testdir = Path(testdir)
fdroidserver.common.config = {'accepted_formats': ['yml']}
# rewrite metadata
allapps = fdroidserver.metadata.read_metadata()
for appid, app in allapps.items():
if appid == 'org.fdroid.fdroid':
fdroidserver.metadata.write_metadata(
testdir / (appid + '.yml'), app
)
# assert rewrite result
self.maxDiff = None
file_name = 'org.fdroid.fdroid.yml'
self.assertEqual(
(testdir / file_name).read_text(encoding='utf-8'),
(Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
)
@mock.patch('git.Repo')
def test_rewrite_yaml_special_build_params(self, git_Repo):
with tempfile.TemporaryDirectory() as testdir:
testdir = Path(testdir)
# rewrite metadata
allapps = fdroidserver.metadata.read_metadata()
for appid, app in allapps.items():
if appid == 'app.with.special.build.params':
fdroidserver.metadata.write_metadata(
testdir / (appid + '.yml'), app
)
# assert rewrite result
self.maxDiff = None
file_name = 'app.with.special.build.params.yml'
self.assertEqual(
(testdir / file_name).read_text(encoding='utf-8'),
(Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
)
def test_normalize_type_string(self):
"""TYPE_STRING currently has some quirky behavior."""
self.assertEqual('123456', metadata._normalize_type_string(123456))
self.assertEqual('1.0', metadata._normalize_type_string(1.0))
self.assertEqual('0', metadata._normalize_type_string(0))
self.assertEqual('0.0', metadata._normalize_type_string(0.0))
self.assertEqual('0.1', metadata._normalize_type_string(0.1))
self.assertEqual('[]', metadata._normalize_type_string(list()))
self.assertEqual('{}', metadata._normalize_type_string(dict()))
self.assertEqual('false', metadata._normalize_type_string(False))
self.assertEqual('true', metadata._normalize_type_string(True))
def test_post_parse_yaml_metadata(self):
yamldata = dict()
metadata.post_parse_yaml_metadata(yamldata)
yamldata[
'AllowedAPKSigningKeys'
] = 'c03dac71394d6c26766f1b04d3e31cfcac5d03b55d8aa40cc9b9fa6b74354c66'
metadata.post_parse_yaml_metadata(yamldata)
def test_post_parse_yaml_metadata_fails(self):
yamldata = {'AllowedAPKSigningKeys': True}
with self.assertRaises(TypeError):
metadata.post_parse_yaml_metadata(yamldata)
def test_post_parse_yaml_metadata_builds(self):
yamldata = OrderedDict()
builds = []
yamldata['Builds'] = builds
build = OrderedDict()
builds.append(build)
build['versionCode'] = 1.1
self.assertRaises(
fdroidserver.exception.MetaDataException,
fdroidserver.metadata.post_parse_yaml_metadata,
yamldata,
)
build['versionCode'] = '1'
self.assertRaises(
fdroidserver.exception.MetaDataException,
fdroidserver.metadata.post_parse_yaml_metadata,
yamldata,
)
build['versionCode'] = 1
build['versionName'] = 1
fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
self.assertNotEqual(1, yamldata['Builds'][0]['versionName'])
self.assertEqual('1', yamldata['Builds'][0]['versionName'])
self.assertEqual(1, yamldata['Builds'][0]['versionCode'])
build['versionName'] = 1.0
fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
self.assertNotEqual(1.0, yamldata['Builds'][0]['versionName'])
self.assertEqual('1.0', yamldata['Builds'][0]['versionName'])
build['commit'] = 1.0
fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
self.assertNotEqual(1.0, yamldata['Builds'][0]['commit'])
self.assertEqual('1.0', yamldata['Builds'][0]['commit'])
teststr = '98234fab134b'
build['commit'] = teststr
fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
self.assertEqual(teststr, yamldata['Builds'][0]['commit'])
testcommitid = 1234567890
build['commit'] = testcommitid
fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
self.assertNotEqual(testcommitid, yamldata['Builds'][0]['commit'])
self.assertEqual('1234567890', yamldata['Builds'][0]['commit'])
def test_read_metadata_sort_by_time(self):
with tempfile.TemporaryDirectory() as testdir, TmpCwd(testdir):
testdir = Path(testdir)
metadatadir = testdir / 'metadata'
metadatadir.mkdir()
randomlist = []
randomapps = list((self.basedir / 'metadata').glob('*.yml'))
random.shuffle(randomapps)
i = 1
for f in randomapps:
shutil.copy(f, metadatadir)
new = metadatadir / f.name
stat = new.stat()
os.utime(new, (stat.st_ctime, stat.st_mtime + i))
# prepend new item so newest is always first
randomlist = [f.stem] + randomlist
i += 1
allapps = fdroidserver.metadata.read_metadata(sort_by_time=True)
allappids = []
for appid, app in allapps.items():
allappids.append(appid)
self.assertEqual(randomlist, allappids)
def test_parse_yaml_metadata_0size_file(self):
self.assertEqual(dict(), metadata.parse_yaml_metadata(_get_mock_mf('')))
def test_parse_yaml_metadata_empty_dict_file(self):
self.assertEqual(dict(), metadata.parse_yaml_metadata(_get_mock_mf('{}')))
def test_parse_yaml_metadata_empty_string_file(self):
self.assertEqual(dict(), metadata.parse_yaml_metadata(_get_mock_mf('""')))
def test_parse_yaml_metadata_fail_on_root_list(self):
with self.assertRaises(MetaDataException):
metadata.parse_yaml_metadata(_get_mock_mf('-'))
with self.assertRaises(MetaDataException):
metadata.parse_yaml_metadata(_get_mock_mf('[]'))
with self.assertRaises(MetaDataException):
metadata.parse_yaml_metadata(_get_mock_mf('- AutoName: fake'))
def test_parse_yaml_metadata_type_list_str(self):
v = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
mf = _get_mock_mf('AllowedAPKSigningKeys: "%s"' % v)
self.assertEqual(
v,
metadata.parse_yaml_metadata(mf)['AllowedAPKSigningKeys'][0],
)
def test_parse_yaml_metadata_type_list_build_str(self):
mf = _get_mock_mf('Builds: [{versionCode: 1, rm: s}]')
self.assertEqual(
metadata.parse_yaml_metadata(mf),
{'Builds': [{'rm': ['s'], 'versionCode': 1}]},
)
def test_parse_yaml_metadata_app_type_list_fails(self):
mf = _get_mock_mf('AllowedAPKSigningKeys: true')
with self.assertRaises(TypeError):
metadata.parse_yaml_metadata(mf)
mf = _get_mock_mf('AllowedAPKSigningKeys: 1')
with self.assertRaises(TypeError):
metadata.parse_yaml_metadata(mf)
mf.name = 'mock_filename.yaml'
self.assertEqual(fdroidserver.metadata.parse_yaml_metadata(mf), dict())
def test_parse_yaml_metadata_unknown_app_field(self):
mf = io.StringIO(
textwrap.dedent(
"""\
AutoName: F-Droid
RepoType: git
Builds: []
bad: value"""
)
)
mf.name = 'mock_filename.yaml'
with self.assertRaises(MetaDataException):
fdroidserver.metadata.parse_yaml_metadata(mf)
def test_parse_yaml_metadata_unknown_build_flag(self):
mf = io.StringIO(
textwrap.dedent(
"""\
AutoName: F-Droid
RepoType: git
Builds:
- bad: value"""
)
)
mf.name = 'mock_filename.yaml'
with self.assertRaises(MetaDataException):
fdroidserver.metadata.parse_yaml_metadata(mf)
def test_parse_yaml_metadata_continue_on_warning(self):
"""When errors are disabled, parsing should provide something that can work.
When errors are disabled, then it should try to give data that
lets something happen. A zero-length file is valid for
operation, it just declares a Application ID as "known" and
nothing else. This example gives a list as the base in the
.yml file, which is unparsable, so it gives a warning message
and carries on with a blank dict.
"""
fdroidserver.metadata.warnings_action = None
mf = _get_mock_mf('[AntiFeatures: Tracking]')
self.assertEqual(fdroidserver.metadata.parse_yaml_metadata(mf), dict())
def test_parse_yaml_srclib_corrupt_file(self):
with tempfile.TemporaryDirectory() as testdir:
testdir = Path(testdir)
srclibfile = testdir / 'srclib/mock.yml'
srclibfile.parent.mkdir()
with srclibfile.open('w') as fp:
fp.write(
textwrap.dedent(
"""
- RepoType: git
- Repo: https://github.com/realm/realm-js.git
"""
)
)
with self.assertRaises(MetaDataException):
fdroidserver.metadata.parse_yaml_srclib(srclibfile)
def test_write_yaml_with_placeholder_values(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app.SourceCode = "https://gitlab.com/fdroid/fdroidclient.git"
app.IssueTracker = "https://gitlab.com/fdroid/fdroidclient/issues"
app.RepoType = 'git'
app.Repo = 'https://gitlab.com/fdroid/fdroidclient.git'
app.AutoUpdateMode = 'None'
app.UpdateCheckMode = 'Tags'
build = fdroidserver.metadata.Build()
build.versionName = 'Unknown' # taken from fdroidserver/import.py
build.versionCode = '0' # taken from fdroidserver/import.py
build.disable = 'Generated by import.py ...'
build.commit = 'Unknown'
build.gradle = [True]
app['Builds'] = [build]
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.assertEqual(
mf.read(),
textwrap.dedent(
"""\
Categories:
- None
License: Unknown
SourceCode: https://gitlab.com/fdroid/fdroidclient.git
IssueTracker: https://gitlab.com/fdroid/fdroidclient/issues
RepoType: git
Repo: https://gitlab.com/fdroid/fdroidclient.git
Builds:
- versionName: Unknown
versionCode: 0
disable: Generated by import.py ...
commit: Unknown
gradle:
- true
AutoUpdateMode: None
UpdateCheckMode: Tags
"""
),
)
def test_parse_yaml_metadata_prebuild_list(self):
mf = io.StringIO(
textwrap.dedent(
"""\
AutoName: F-Droid
RepoType: git
Builds:
- versionCode: 1
versionName: v0.1.0
sudo:
- apt-get update
- apt-get install -y whatever
- sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
init:
- bash generate_some_file.sh
- sed -i -e 'g/what/ever/' /some/file
prebuild:
- npm something
- echo 'important setting' >> /a/file
build:
- ./gradlew someSpecialTask
- sed -i 'd/that wrong config/' gradle.properties
- ./gradlew compile
"""
)
)
mf.name = 'mock_filename.yaml'
mf.seek(0)
result = fdroidserver.metadata.parse_yaml_metadata(mf)
self.maxDiff = None
self.assertDictEqual(
result,
{
'AutoName': 'F-Droid',
'RepoType': 'git',
'Builds': [
{
'versionCode': 1,
'versionName': 'v0.1.0',
'sudo': [
"apt-get update",
"apt-get install -y whatever",
"sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
],
'init': [
"bash generate_some_file.sh",
"sed -i -e 'g/what/ever/' /some/file",
],
'prebuild': [
"npm something",
"echo 'important setting' >> /a/file",
],
'build': [
"./gradlew someSpecialTask",
"sed -i 'd/that wrong config/' gradle.properties",
"./gradlew compile",
],
}
],
},
)
def test_parse_yaml_metadata_prebuild_strings(self):
mf = io.StringIO(
textwrap.dedent(
"""\
AutoName: F-Droid
RepoType: git
Builds:
- versionCode: 1
versionName: v0.1.0
sudo: |-
apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
init: bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file
prebuild: npm something && echo 'important setting' >> /a/file
build: |-
./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile
"""
)
)
mf.name = 'mock_filename.yaml'
mf.seek(0)
result = fdroidserver.metadata.parse_yaml_metadata(mf)
self.maxDiff = None
self.assertDictEqual(
result,
{
'AutoName': 'F-Droid',
'RepoType': 'git',
'Builds': [
{
'versionCode': 1,
'versionName': 'v0.1.0',
'sudo': [
"apt-get update && "
"apt-get install -y whatever && "
"sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"
],
'init': [
"bash generate_some_file.sh && "
"sed -i -e 'g/what/ever/' /some/file"
],
'prebuild': [
"npm something && echo 'important setting' >> /a/file"
],
'build': [
"./gradlew someSpecialTask && "
"sed -i 'd/that wrong config/' gradle.properties && "
"./gradlew compile"
],
}
],
},
)
def test_parse_yaml_metadata_prebuild_string(self):
mf = io.StringIO(
textwrap.dedent(
"""\
AutoName: F-Droid
RepoType: git
Builds:
- versionCode: 1
versionName: v0.1.0
prebuild: |-
a && b && sed -i 's,a,b,'
"""
)
)
mf.name = 'mock_filename.yaml'
mf.seek(0)
result = fdroidserver.metadata.parse_yaml_metadata(mf)
self.assertDictEqual(
result,
{
'AutoName': 'F-Droid',
'RepoType': 'git',
'Builds': [
{
'versionCode': 1,
'versionName': 'v0.1.0',
'prebuild': ["a && b && " "sed -i 's,a,b,'"],
}
],
},
)
def test_parse_yaml_provides_should_be_ignored(self):
mf = io.StringIO(
textwrap.dedent(
"""\
Provides: this.is.deprecated
AutoName: F-Droid
RepoType: git
Builds:
- versionCode: 1
versionName: v0.1.0
prebuild: |-
a && b && sed -i 's,a,b,'
"""
)
)
mf.name = 'mock_filename.yaml'
mf.seek(0)
result = fdroidserver.metadata.parse_yaml_metadata(mf)
self.assertNotIn('Provides', result)
self.assertNotIn('provides', result)
def test_write_yaml_1_line_scripts_as_string(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app['Builds'] = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.sudo = ["chmod +rwx /opt"]
build.init = ["sed -i -e 'g/what/ever/' /some/file"]
build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"]
build.build = ["./gradlew compile"]
app['Builds'].append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.assertEqual(
mf.read(),
textwrap.dedent(
"""\
Categories:
- None
License: Unknown
Builds:
- versionName: v1.2.3
versionCode: 102030
sudo: chmod +rwx /opt
init: sed -i -e 'g/what/ever/' /some/file
prebuild: sed -i 'd/that wrong config/' gradle.properties
build: ./gradlew compile
AutoUpdateMode: None
UpdateCheckMode: None
"""
),
)
def test_write_yaml_1_line_scripts_as_list(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app['Builds'] = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.sudo = ["chmod +rwx /opt"]
build.init = ["sed -i -e 'g/what/ever/' /some/file"]
build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"]
build.build = ["./gradlew compile"]
app['Builds'].append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.assertEqual(
mf.read(),
textwrap.dedent(
"""\
Categories:
- None
License: Unknown
Builds:
- versionName: v1.2.3
versionCode: 102030
sudo: chmod +rwx /opt
init: sed -i -e 'g/what/ever/' /some/file
prebuild: sed -i 'd/that wrong config/' gradle.properties
build: ./gradlew compile
AutoUpdateMode: None
UpdateCheckMode: None
"""
),
)
def test_write_yaml_multiline_scripts_from_list(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app['Builds'] = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.sudo = [
"apt-get update",
"apt-get install -y whatever",
"sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
]
build.init = [
"bash generate_some_file.sh",
"sed -i -e 'g/what/ever/' /some/file",
]
build.prebuild = ["npm something", "echo 'important setting' >> /a/file"]
build.build = [
"./gradlew someSpecialTask",
"sed -i 'd/that wrong config/' gradle.properties",
"./gradlew compile",
]
app['Builds'].append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.assertEqual(
mf.read(),
textwrap.dedent(
"""\
Categories:
- None
License: Unknown
Builds:
- versionName: v1.2.3
versionCode: 102030
sudo:
- apt-get update
- apt-get install -y whatever
- sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
init:
- bash generate_some_file.sh
- sed -i -e 'g/what/ever/' /some/file
prebuild:
- npm something
- echo 'important setting' >> /a/file
build:
- ./gradlew someSpecialTask
- sed -i 'd/that wrong config/' gradle.properties
- ./gradlew compile
AutoUpdateMode: None
UpdateCheckMode: None
"""
),
)
def test_write_yaml_multiline_scripts_from_string(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app['Builds'] = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.sudo = [
"apt-get update",
"apt-get install -y whatever",
"sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
]
build.init = [
"bash generate_some_file.sh",
"sed -i -e 'g/what/ever/' /some/file",
]
build.prebuild = ["npm something", "echo 'important setting' >> /a/file"]
build.build = [
"./gradlew someSpecialTask",
"sed -i 'd/that wrong config/' gradle.properties",
"./gradlew compile",
]
app['Builds'].append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.assertEqual(
mf.read(),
textwrap.dedent(
"""\
Categories:
- None
License: Unknown
Builds:
- versionName: v1.2.3
versionCode: 102030
sudo:
- apt-get update
- apt-get install -y whatever
- sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
init:
- bash generate_some_file.sh
- sed -i -e 'g/what/ever/' /some/file
prebuild:
- npm something
- echo 'important setting' >> /a/file
build:
- ./gradlew someSpecialTask
- sed -i 'd/that wrong config/' gradle.properties
- ./gradlew compile
AutoUpdateMode: None
UpdateCheckMode: None
"""
),
)
def test_write_yaml_make_sure_provides_does_not_get_written(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app.Provides = 'this.is.deprecated'
app['Builds'] = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.gradle = ['yes']
app['Builds'].append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.assertEqual(
mf.read(),
textwrap.dedent(
"""\
Categories:
- None
License: Unknown
Builds:
- versionName: v1.2.3
versionCode: 102030
gradle:
- yes
AutoUpdateMode: None
UpdateCheckMode: None
"""
),
)
def test_parse_yaml_srclib_unknown_key(self):
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
with Path('test.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
RepoType: git
Repo: https://example.com/test.git
Evil: I should not be here.
'''
)
)
with self.assertRaisesRegex(
MetaDataException,
"Invalid srclib metadata: " "unknown key 'Evil' in " "'test.yml'",
):
fdroidserver.metadata.parse_yaml_srclib(Path('test.yml'))
def test_parse_yaml_srclib_does_not_exists(self):
with self.assertRaisesRegex(
MetaDataException,
"Invalid scrlib metadata: "
r"'non(/|\\)existent-test-srclib.yml' "
"does not exist",
):
fdroidserver.metadata.parse_yaml_srclib(
Path('non/existent-test-srclib.yml')
)
def test_parse_yaml_srclib_simple(self):
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
with Path('simple.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
# this should be simple
RepoType: git
Repo: https://git.host/repo.git
'''
)
)
srclib = fdroidserver.metadata.parse_yaml_srclib(Path('simple.yml'))
self.assertDictEqual(
{
'Repo': 'https://git.host/repo.git',
'RepoType': 'git',
'Subdir': None,
'Prepare': None,
},
srclib,
)
def test_parse_yaml_srclib_simple_with_blanks(self):
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
with Path('simple.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
# this should be simple
RepoType: git
Repo: https://git.host/repo.git
Subdir:
Prepare:
'''
)
)
srclib = fdroidserver.metadata.parse_yaml_srclib(Path('simple.yml'))
self.assertDictEqual(
{
'Repo': 'https://git.host/repo.git',
'RepoType': 'git',
'Subdir': [''],
'Prepare': [],
},
srclib,
)
def test_parse_yaml_srclib_Changelog_cketti(self):
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
with Path('Changelog-cketti.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
RepoType: git
Repo: https://github.com/cketti/ckChangeLog
Subdir: library,ckChangeLog/src/main
Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties"
'''
)
)
srclib = fdroidserver.metadata.parse_yaml_srclib(
Path('Changelog-cketti.yml')
)
self.assertDictEqual(
srclib,
{
'Repo': 'https://github.com/cketti/ckChangeLog',
'RepoType': 'git',
'Subdir': ['library', 'ckChangeLog/src/main'],
'Prepare': [
"[ -f project.properties ] || echo 'source.dir=java' > "
"ant.properties && echo -e "
"'android.library=true\\ntarget=android-19' > project.properties"
],
},
)
def test_read_srclibs_yml_subdir_list(self):
fdroidserver.metadata.srclibs = None
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
Path('srclibs').mkdir()
with Path('srclibs/with-list.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
# this should be simple
RepoType: git
Repo: https://git.host/repo.git
Subdir:
- This is your last chance.
- After this, there is no turning back.
- You take the blue pill—the story ends,
- you wake up in your bed
- and believe whatever you want to believe.
- You take the red pill—you stay in Wonderland
- and I show you how deep the rabbit-hole goes.
Prepare:
There is a difference between knowing the path
and walking the path.
'''
)
)
fdroidserver.metadata.read_srclibs()
self.maxDiff = None
self.assertDictEqual(
fdroidserver.metadata.srclibs,
{
'with-list': {
'RepoType': 'git',
'Repo': 'https://git.host/repo.git',
'Subdir': [
'This is your last chance.',
'After this, there is no turning back.',
'You take the blue pill—the story ends,',
'you wake up in your bed',
'and believe whatever you want to believe.',
'You take the red pill—you stay in Wonderland',
'and I show you how deep the rabbit-hole goes.',
],
'Prepare': [
'There is a difference between knowing the path '
'and walking the path.'
],
}
},
)
def test_read_srclibs_yml_prepare_list(self):
fdroidserver.metadata.srclibs = None
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
Path('srclibs').mkdir()
with Path('srclibs/with-list.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
# this should be simple
RepoType: git
Repo: https://git.host/repo.git
Subdir:
Prepare:
- Many
- invalid
- commands
- here.
'''
)
)
fdroidserver.metadata.read_srclibs()
self.maxDiff = None
self.assertDictEqual(
fdroidserver.metadata.srclibs,
{
'with-list': {
'RepoType': 'git',
'Repo': 'https://git.host/repo.git',
'Subdir': [''],
'Prepare': [
'Many',
'invalid',
'commands',
'here.',
],
}
},
)
def test_read_srclibs(self):
fdroidserver.metadata.srclibs = None
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
Path('srclibs').mkdir()
with Path('srclibs/simple.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
RepoType: git
Repo: https://git.host/repo.git
'''
)
)
with Path('srclibs/simple-wb.yml').open('w', encoding='utf-8') as f:
f.write(
textwrap.dedent(
'''\
# this should be simple
RepoType: git
Repo: https://git.host/repo.git
Subdir:
Prepare:
'''
)
)
fdroidserver.metadata.read_srclibs()
self.assertDictEqual(
fdroidserver.metadata.srclibs,
{
'simple-wb': {
'RepoType': 'git',
'Repo': 'https://git.host/repo.git',
'Subdir': [''],
'Prepare': [],
},
'simple': {
'RepoType': 'git',
'Repo': 'https://git.host/repo.git',
'Subdir': None,
'Prepare': None,
},
},
)
def test_build_ndk_path(self):
""""""
with tempfile.TemporaryDirectory(prefix='android-sdk-') as sdk_path:
config = {'ndk_paths': {}, 'sdk_path': sdk_path}
fdroidserver.common.config = config
build = fdroidserver.metadata.Build()
build.ndk = 'r10e'
self.assertEqual('', build.ndk_path())
correct = '/fake/path/ndk/r21b'
config['ndk_paths'] = {'r21b': correct}
self.assertEqual('', build.ndk_path())
config['ndk_paths'] = {'r10e': correct}
self.assertEqual(correct, build.ndk_path())
r10e = '/fake/path/ndk/r10e'
r22b = '/fake/path/ndk/r22e'
config['ndk_paths'] = {'r10e': r10e, 'r22b': r22b}
self.assertEqual(r10e, build.ndk_path())
build.ndk = ['r10e', 'r22b']
self.assertEqual(r10e, build.ndk_path())
build.ndk = ['r22b', 'r10e']
self.assertEqual(r22b, build.ndk_path())
def test_build_ndk_path_only_accepts_str(self):
"""Paths in the config must be strings, never pathlib.Path instances"""
config = {'ndk_paths': {'r24': Path('r24')}}
fdroidserver.common.config = config
build = fdroidserver.metadata.Build()
build.ndk = 'r24'
with self.assertRaises(TypeError):
build.ndk_path()
class PostMetadataParseTest(unittest.TestCase):
"""Test the functions that post process the YAML input.
The following series of "post_metadata_parse" tests map out the
current state of automatic type conversion in the YAML post
processing. They are not necessary a statement of how things
should be, but more to surface the details of it functions.
"""
def setUp(self):
fdroidserver.metadata.warnings_action = 'error'
def _post_metadata_parse_app_list(self, from_yaml, expected):
app = {'AntiFeatures': from_yaml}
metadata.post_parse_yaml_metadata(app)
return {'AntiFeatures': expected}, app
def _post_metadata_parse_app_string(self, from_yaml, expected):
app = {'Repo': from_yaml}
metadata.post_parse_yaml_metadata(app)
return {'Repo': expected}, app
def _post_metadata_parse_build_bool(self, from_yaml, expected):
tested_key = 'submodules'
app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]}
post = copy.deepcopy(app)
metadata.post_parse_yaml_metadata(post)
del app['Builds'][0]['versionCode']
del post['Builds'][0]['versionCode']
for build in post['Builds']:
for k in list(build):
if k != tested_key:
del build[k]
app['Builds'][0][tested_key] = expected
return app, post
def _post_metadata_parse_build_int(self, from_yaml, expected):
tested_key = 'versionCode'
app = {'Builds': [{'versionCode': from_yaml}]}
post = copy.deepcopy(app)
metadata.post_parse_yaml_metadata(post)
for build in post['Builds']:
for k in list(build):
if k != tested_key:
del build[k]
app['Builds'][0][tested_key] = expected
return app, post
def _post_metadata_parse_build_list(self, from_yaml, expected):
tested_key = 'rm'
app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]}
post = copy.deepcopy(app)
metadata.post_parse_yaml_metadata(post)
del app['Builds'][0]['versionCode']
del post['Builds'][0]['versionCode']
for build in post['Builds']:
for k in list(build):
if k != tested_key:
del build[k]
app['Builds'][0][tested_key] = expected
return app, post
def _post_metadata_parse_build_script(self, from_yaml, expected):
tested_key = 'build'
app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]}
post = copy.deepcopy(app)
metadata.post_parse_yaml_metadata(post)
del app['Builds'][0]['versionCode']
del post['Builds'][0]['versionCode']
for build in post['Builds']:
for k in list(build):
if k != tested_key:
del build[k]
app['Builds'][0][tested_key] = expected
return app, post
def _post_metadata_parse_build_string(self, from_yaml, expected):
tested_key = 'commit'
app = {'Builds': [{'versionCode': 1, tested_key: from_yaml}]}
post = copy.deepcopy(app)
metadata.post_parse_yaml_metadata(post)
del app['Builds'][0]['versionCode']
del post['Builds'][0]['versionCode']
for build in post['Builds']:
for k in list(build):
if k != tested_key:
del build[k]
app['Builds'][0][tested_key] = expected
return app, post
def test_post_metadata_parse_int(self):
"""Run the int 123456 through the various field and flag types."""
with self.assertRaises(TypeError):
self._post_metadata_parse_app_list(123456, TypeError)
self.assertEqual(*self._post_metadata_parse_app_string(123456, '123456'))
self.assertEqual(*self._post_metadata_parse_build_bool(123456, 123456))
self.assertEqual(*self._post_metadata_parse_build_int(123456, 123456))
self.assertEqual(*self._post_metadata_parse_build_list(123456, ['123456']))
self.assertEqual(*self._post_metadata_parse_build_script(123456, ['123456']))
self.assertEqual(*self._post_metadata_parse_build_string(123456, '123456'))
def test_post_metadata_parse_int_0(self):
"""Run the int 0 through the various field and flag types."""
self.assertEqual(*self._post_metadata_parse_app_list(0, 0))
self.assertEqual(*self._post_metadata_parse_app_string(0, '0'))
self.assertEqual(*self._post_metadata_parse_build_bool(0, 0))
self.assertEqual(*self._post_metadata_parse_build_int(0, 0))
self.assertEqual(*self._post_metadata_parse_build_list(0, ['0']))
self.assertEqual(*self._post_metadata_parse_build_script(0, ['0']))
self.assertEqual(*self._post_metadata_parse_build_string(0, '0'))
def test_post_metadata_parse_float_0_0(self):
"""Run the float 0.0 through the various field and flag types."""
self.assertEqual(*self._post_metadata_parse_app_list(0.0, 0.0))
self.assertEqual(*self._post_metadata_parse_app_string(0.0, '0.0'))
self.assertEqual(*self._post_metadata_parse_build_bool(0.0, 0.0))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int(0.0, MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list(0.0, 0.0))
self.assertEqual(*self._post_metadata_parse_build_script(0.0, 0.0))
self.assertEqual(*self._post_metadata_parse_build_string(0.0, '0.0'))
def test_post_metadata_parse_float_0_1(self):
"""Run the float 0.1 through the various field and flag types."""
with self.assertRaises(TypeError):
self._post_metadata_parse_app_list(0.1, TypeError)
self.assertEqual(*self._post_metadata_parse_app_string(0.1, '0.1'))
self.assertEqual(*self._post_metadata_parse_build_bool(0.1, 0.1))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int(0.1, MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list(0.1, 0.1))
self.assertEqual(*self._post_metadata_parse_build_script(0.1, 0.1))
self.assertEqual(*self._post_metadata_parse_build_string(0.1, '0.1'))
def test_post_metadata_parse_float_1_0(self):
"""Run the float 1.0 through the various field and flag types."""
with self.assertRaises(TypeError):
self._post_metadata_parse_app_list(1.0, TypeError)
self.assertEqual(*self._post_metadata_parse_app_string(1.0, '1.0'))
self.assertEqual(*self._post_metadata_parse_build_bool(1.0, 1.0))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int(1.0, MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list(1.0, 1.0))
self.assertEqual(*self._post_metadata_parse_build_script(1.0, 1.0))
self.assertEqual(*self._post_metadata_parse_build_string(1.0, '1.0'))
def test_post_metadata_parse_empty_list(self):
self.assertEqual(*self._post_metadata_parse_app_list(list(), list()))
self.assertEqual(*self._post_metadata_parse_app_string(list(), list()))
self.assertEqual(*self._post_metadata_parse_build_bool(list(), list()))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int(list(), MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list(list(), list()))
self.assertEqual(*self._post_metadata_parse_build_script(list(), list()))
self.assertEqual(*self._post_metadata_parse_build_string(list(), list()))
def test_post_metadata_parse_set_of_1(self):
self.assertEqual(*self._post_metadata_parse_app_list({1}, ['1']))
self.assertEqual(*self._post_metadata_parse_app_string({1}, '{1}'))
self.assertEqual(*self._post_metadata_parse_build_bool({1}, {1}))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int({1}, MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list({1}, {1}))
self.assertEqual(*self._post_metadata_parse_build_script({1}, {1}))
self.assertEqual(*self._post_metadata_parse_build_string({1}, '{1}'))
def test_post_metadata_parse_empty_dict(self):
self.assertEqual(*self._post_metadata_parse_app_list(dict(), dict()))
self.assertEqual(*self._post_metadata_parse_app_string(dict(), dict()))
self.assertEqual(*self._post_metadata_parse_build_bool(dict(), dict()))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int(dict(), MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list(dict(), dict()))
self.assertEqual(*self._post_metadata_parse_build_script(dict(), dict()))
self.assertEqual(*self._post_metadata_parse_build_string(dict(), dict()))
def test_post_metadata_parse_list_int_string(self):
self.assertEqual(*self._post_metadata_parse_app_list([1, 'a'], ['1', 'a']))
self.assertEqual(*self._post_metadata_parse_app_string([1, 'a'], "[1, 'a']"))
self.assertEqual(*self._post_metadata_parse_build_bool([1, 'a'], [1, 'a']))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int([1, 'a'], MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list([1, 'a'], [1, 'a']))
self.assertEqual(*self._post_metadata_parse_build_script([1, 'a'], [1, 'a']))
self.assertEqual(*self._post_metadata_parse_build_string([1, 'a'], "[1, 'a']"))
def test_post_metadata_parse_dict_int_string(self):
self.assertEqual(*self._post_metadata_parse_app_list({'k': 1}, ['k']))
self.assertEqual(*self._post_metadata_parse_app_string({'k': 1}, "{'k': 1}"))
self.assertEqual(*self._post_metadata_parse_build_bool({'k': 1}, {'k': 1}))
with self.assertRaises(MetaDataException):
self._post_metadata_parse_build_int({'k': 1}, MetaDataException)
self.assertEqual(*self._post_metadata_parse_build_list({'k': 1}, {'k': 1}))
self.assertEqual(*self._post_metadata_parse_build_script({'k': 1}, {'k': 1}))
self.assertEqual(*self._post_metadata_parse_build_string({'k': 1}, "{'k': 1}"))
def test_post_metadata_parse_false(self):
self.assertEqual(*self._post_metadata_parse_app_list(False, False))
self.assertEqual(*self._post_metadata_parse_app_string(False, 'false'))
self.assertEqual(*self._post_metadata_parse_build_bool(False, False))
self.assertEqual(*self._post_metadata_parse_build_int(False, False))
self.assertEqual(*self._post_metadata_parse_build_list(False, ['false']))
self.assertEqual(*self._post_metadata_parse_build_script(False, ['false']))
self.assertEqual(*self._post_metadata_parse_build_string(False, 'false'))
def test_post_metadata_parse_true(self):
with self.assertRaises(TypeError):
self._post_metadata_parse_app_list(True, TypeError)
self.assertEqual(*self._post_metadata_parse_app_string(True, 'true'))
self.assertEqual(*self._post_metadata_parse_build_bool(True, True))
self.assertEqual(*self._post_metadata_parse_build_int(True, True))
self.assertEqual(*self._post_metadata_parse_build_list(True, ['true']))
self.assertEqual(*self._post_metadata_parse_build_script(True, ['true']))
self.assertEqual(*self._post_metadata_parse_build_string(True, 'true'))
if __name__ == "__main__":
parser = optparse.OptionParser()
parser.add_option(
"-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal",
)
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(MetadataTest))
unittest.main(failfast=False)