fdroidserver/tests/metadata.TestCase

#!/usr/bin/env python3

# http://www.drdobbs.com/testing/unit-testing-with-python/240165163

import io
import glob
import inspect
import logging
import optparse
import os
import random
import shutil
import sys
import unittest
import yaml
import tempfile
import textwrap
from collections import OrderedDict
from unittest import mock

localmodule = os.path.realpath(
    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
print('localmodule: ' + localmodule)
if localmodule not in sys.path:
    sys.path.insert(0, localmodule)

from testcommon import TmpCwd

import fdroidserver.common
import fdroidserver.metadata
from fdroidserver.exception import MetaDataException


class MetadataTest(unittest.TestCase):
    '''fdroidserver/metadata.py'''

    def setUp(self):
        logging.basicConfig(level=logging.DEBUG)
        self.basedir = os.path.join(localmodule, 'tests')
        self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
        if not os.path.exists(self.tmpdir):
            os.makedirs(self.tmpdir)
        os.chdir(self.basedir)

    def test_FieldValidator_BitcoinAddress(self):
        validator = None
        for vali in fdroidserver.metadata.valuetypes:
            if vali.name == 'Bitcoin address':
                validator = vali
                break
        self.assertIsNotNone(validator, "could not find 'Bitcoin address' validator")

        fdroidserver.metadata.warnings_action = 'error'

        # some valid addresses (P2PKH, P2SH, Bech32)
        self.assertIsNone(validator.check('1BrrrrErsrWetrTrnrrrrm4GFg7xJaNVN2', 'fake.app.id'))
        self.assertIsNone(validator.check('3JrrrrWrEZr3rNrrvrecrnyirrnqRhWNLy', 'fake.app.id'))
        self.assertIsNone(validator.check('bc1qar0srrr7xrkvr5lr43lrdnwrre5rgtrzrf5rrq', 'fake.app.id'))

        # some invalid addresses
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '21BvMrSYsrWrtrrlL5A10mlGFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '5Hrgr3ur5rGLrfKrrrrrrHSrqJrroGrrzrQrrrrrrLNrsrDrrrA', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'L1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')

    def test_FieldValidator_LitecoinAddress(self):
        validator = None
        for vali in fdroidserver.metadata.valuetypes:
            if vali.name == 'Litecoin address':
                validator = vali
                break
        self.assertIsNotNone(validator, "could not find 'Litecoin address' validator")

        fdroidserver.metadata.warnings_action = 'error'

        # some valid addresses (L, M, 3)
        self.assertIsNone(validator.check('LgeGrrrrJAxyXprrPrrBrrX5Qrrrrrrrrd', 'fake.app.id'))
        self.assertIsNone(validator.check('MrrrrrrrJAxyXpanPtrrRAX5QHxvUJo8id', 'fake.app.id'))
        self.assertIsNone(validator.check('3rereVr9rAryrranrrrrrAXrrHx', 'fake.app.id'))

        # some invalid addresses (various special use/testnet addresses, invalid chars)
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '21BvMrSYsrWrtrrrn5Au4l4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '5Hrgr3ur5rGLrfKrrrrrr1SrqJrroGrrzrQrrrrrrLNrsrDrrrA', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'L0000rSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')

    def test_valid_funding_yml_regex(self):
        """Check the regex can find all the cases"""
        with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
            data = yaml.safe_load(fp)

        for k, entries in data.items():
            for entry in entries:
                m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry)
                if k == 'custom':
                    pass
                elif k == 'bad':
                    self.assertIsNone(m, 'this is an invalid %s username: {%s}' % (k, entry))
                else:
                    self.assertIsNotNone(m, 'this is a valid %s username: {%s}' % (k, entry))

    def test_read_metadata(self):

        def _build_yaml_representer(dumper, data):
            '''Creates a YAML representation of a Build instance'''
            return dumper.represent_dict(data)

        self.maxDiff = None

        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.metadata.warnings_action = None

        apps = fdroidserver.metadata.read_metadata(xref=True)
        for appid in ('org.smssecure.smssecure', 'org.adaway',
                      'org.videolan.vlc', 'com.politedroid'):
            savepath = os.path.join('metadata', 'dump', appid + '.yaml')
            frommeta = dict(apps[appid])
            self.assertTrue(appid in apps)
            with open(savepath, 'r') as f:
                frompickle = yaml.load(f)
            self.assertEqual(frommeta, frompickle)
            # comment above assert and uncomment below to update test
            # files when new metadata fields are added
            # with open(savepath, 'w') as f:
            #     yaml.add_representer(fdroidserver.metadata.Build, _build_yaml_representer)
            #     yaml.dump(frommeta, f, default_flow_style=False)

    def test_rewrite_yaml_fakeotaupdate(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        fdroidserver.common.config = {'accepted_formats': ['yml']}
        fdroidserver.metadata.warnings_action = None

        # rewrite metadata
        allapps = fdroidserver.metadata.read_metadata(xref=True)
        for appid, app in allapps.items():
            if appid == 'fake.ota.update':
                fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)

        # assert rewrite result
        with open(os.path.join(testdir, 'fake.ota.update.yml'), 'r') as result:
            with open('metadata-rewrite-yml/fake.ota.update.yml', 'r') as orig:
                self.maxDiff = None
                self.assertEqual(result.read(), orig.read())

    def test_rewrite_yaml_fdroidclient(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        fdroidserver.common.config = {'accepted_formats': ['yml']}

        # rewrite metadata
        allapps = fdroidserver.metadata.read_metadata(xref=True)
        for appid, app in allapps.items():
            if appid == 'org.fdroid.fdroid':
                fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)

        # assert rewrite result
        with open(os.path.join(testdir, 'org.fdroid.fdroid.yml'), 'r') as result:
            with open('metadata-rewrite-yml/org.fdroid.fdroid.yml', 'r') as orig:
                self.maxDiff = None
                self.assertEqual(result.read(), orig.read())

    def test_rewrite_yaml_special_build_params(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)

        # rewrite metadata
        allapps = fdroidserver.metadata.read_metadata(xref=True)
        for appid, app in allapps.items():
            if appid == 'app.with.special.build.params':
                fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)

        # assert rewrite result
        with open(os.path.join(testdir, 'app.with.special.build.params.yml'), 'r') as result:
            with open('metadata-rewrite-yml/app.with.special.build.params.yml', 'r') as orig:
                self.maxDiff = None
                self.assertEqual(result.read(), orig.read())

    def test_post_parse_yaml_metadata(self):
        fdroidserver.metadata.warnings_action = 'error'
        yamldata = OrderedDict()
        builds = []
        yamldata['Builds'] = builds
        build = OrderedDict()
        builds.append(build)

        build['versionCode'] = 1.1
        self.assertRaises(fdroidserver.exception.MetaDataException,
                          fdroidserver.metadata.post_parse_yaml_metadata, yamldata)

        build['versionCode'] = '1'
        self.assertRaises(fdroidserver.exception.MetaDataException,
                          fdroidserver.metadata.post_parse_yaml_metadata, yamldata)

        build['versionCode'] = 1
        build['versionName'] = 1
        fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
        self.assertNotEqual(1, yamldata['Builds'][0]['versionName'])
        self.assertEqual('1', yamldata['Builds'][0]['versionName'])
        self.assertEqual(1, yamldata['Builds'][0]['versionCode'])

        build['versionName'] = 1.0
        fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
        self.assertNotEqual(1.0, yamldata['Builds'][0]['versionName'])
        self.assertEqual('1.0', yamldata['Builds'][0]['versionName'])

        build['commit'] = 1.0
        fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
        self.assertNotEqual(1.0, yamldata['Builds'][0]['commit'])
        self.assertEqual('1.0', yamldata['Builds'][0]['commit'])

        teststr = '98234fab134b'
        build['commit'] = teststr
        fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
        self.assertEqual(teststr, yamldata['Builds'][0]['commit'])

        testcommitid = 1234567890
        build['commit'] = testcommitid
        fdroidserver.metadata.post_parse_yaml_metadata(yamldata)
        self.assertNotEqual(testcommitid, yamldata['Builds'][0]['commit'])
        self.assertEqual('1234567890', yamldata['Builds'][0]['commit'])

    def test_read_metadata_sort_by_time(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        metadatadir = os.path.join(testdir, 'metadata')
        os.makedirs(metadatadir)

        randomlist = []
        randomapps = glob.glob(os.path.join(self.basedir, 'metadata', '*.yml'))
        random.shuffle(randomapps)
        i = 1
        for f in randomapps:
            shutil.copy(f, metadatadir)
            new = os.path.join(metadatadir, os.path.basename(f))
            stat = os.stat(new)
            os.utime(new, (stat.st_ctime, stat.st_mtime + i))
            # prepend new item so newest is always first
            randomlist = [os.path.basename(f)[:-4]] + randomlist
            i += 1
        os.chdir(testdir)
        allapps = fdroidserver.metadata.read_metadata(xref=True, sort_by_time=True)
        allappids = []
        for appid, app in allapps.items():
            allappids.append(appid)
        self.assertEqual(randomlist, allappids)

    def test_parse_yaml_metadata_unknown_app_field(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds: []
            bad: value"""))
        mf.name = 'mock_filename.yaml'
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            with self.assertRaises(MetaDataException):
                fdroidserver.metadata.parse_yaml_metadata(mf, {})

    def test_parse_yaml_metadata_unknown_build_flag(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
              - bad: value"""))
        mf.name = 'mock_filename.yaml'
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            with self.assertRaises(MetaDataException):
                fdroidserver.metadata.parse_yaml_metadata(mf, {})

    def test_parse_yaml_srclib_corrupt_file(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        srclibfile = os.path.join(testdir, 'srclib', 'mock.yml')
        os.mkdir(os.path.dirname(srclibfile))
        with open(srclibfile, 'w') as fp:
            fp.write(textwrap.dedent("""
            - RepoType: git
            - Repo: https://github.com/realm/realm-js.git
            """))
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            with self.assertRaises(MetaDataException):
                fdroidserver.metadata.parse_yaml_srclib(srclibfile)

    def test_write_yaml_with_placeholder_values(self):
        mf = io.StringIO()

        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.SourceCode = "https://gitlab.com/fdroid/fdroidclient.git"
        app.IssueTracker = "https://gitlab.com/fdroid/fdroidclient/issues"
        app.RepoType = 'git'
        app.Repo = 'https://gitlab.com/fdroid/fdroidclient.git'
        app.AutoUpdateMode = 'None'
        app.UpdateCheckMode = 'Tags'
        build = fdroidserver.metadata.Build()
        build.versionName = 'Unknown'  # taken from fdroidserver/import.py
        build.versionCode = '0'  # taken from fdroidserver/import.py
        build.disable = 'Generated by import.py ...'
        build.commit = 'Unknown'
        build.gradle = [True]
        app.builds = [build]

        fdroidserver.metadata.write_yaml(mf, app)

        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown
            SourceCode: https://gitlab.com/fdroid/fdroidclient.git
            IssueTracker: https://gitlab.com/fdroid/fdroidclient/issues

            RepoType: git
            Repo: https://gitlab.com/fdroid/fdroidclient.git

            Builds:
              - versionName: Unknown
                versionCode: 0
                disable: Generated by import.py ...
                commit: Unknown
                gradle:
                  - true

            AutoUpdateMode: None
            UpdateCheckMode: Tags
            """))

    def test_parse_yaml_metadata_prebuild_list(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  sudo:
                    - apt-get update
                    - apt-get install -y whatever
                    - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                  init:
                    - bash generate_some_file.sh
                    - sed -i -e 'g/what/ever/' /some/file
                  prebuild:
                    - npm something
                    - echo 'important setting' >> /a/file
                  build:
                    - ./gradlew someSpecialTask
                    - sed -i 'd/that wrong config/' gradle.properties
                    - ./gradlew compile
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
        self.maxDiff = None
        self.assertDictEqual(result, {'AutoName': 'F-Droid',
                                      'RepoType': 'git',
                                      'Builds': [{'versionCode': 1,
                                                  'versionName': 'v0.1.0',
                                                  'sudo': "apt-get update && "
                                                          "apt-get install -y whatever && "
                                                          "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
                                                  'init': "bash generate_some_file.sh && "
                                                          "sed -i -e 'g/what/ever/' /some/file",
                                                  'prebuild': "npm something && echo 'important setting' >> /a/file",
                                                  'build': "./gradlew someSpecialTask && "
                                                           "sed -i 'd/that wrong config/' gradle.properties && "
                                                           "./gradlew compile"}]})

    def test_parse_yaml_metadata_prebuild_strings(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  sudo: |-
                    apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                  init: bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file
                  prebuild: npm something && echo 'important setting' >> /a/file
                  build: |-
                    ./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
        self.maxDiff = None
        self.assertDictEqual(result, {'AutoName': 'F-Droid',
                                      'RepoType': 'git',
                                      'Builds': [{'versionCode': 1,
                                                  'versionName': 'v0.1.0',
                                                  'sudo': "apt-get update && "
                                                          "apt-get install -y whatever && "
                                                          "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
                                                  'init': "bash generate_some_file.sh && "
                                                          "sed -i -e 'g/what/ever/' /some/file",
                                                  'prebuild': "npm something && echo 'important setting' >> /a/file",
                                                  'build': "./gradlew someSpecialTask && "
                                                           "sed -i 'd/that wrong config/' gradle.properties && "
                                                           "./gradlew compile"}]})

    def test_parse_yaml_metadata_prebuild_string(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  prebuild: |-
                    a && b && sed -i 's,a,b,'
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
        self.assertDictEqual(result, {'AutoName': 'F-Droid',
                                      'RepoType': 'git',
                                      'Builds': [{'versionCode': 1,
                                                  'versionName': 'v0.1.0',
                                                  'prebuild': "a && b && "
                                                              "sed -i 's,a,b,'"}]})

    def test_parse_yaml_provides_should_be_ignored(self):
        mf = io.StringIO(textwrap.dedent("""\
            Provides: this.is.deprecated
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  prebuild: |-
                    a && b && sed -i 's,a,b,'
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
            self.assertNotIn('Provides', result)
            self.assertNotIn('provides', result)

    def test_write_yaml_1_line_scripts_as_string(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = "chmod +rwx /opt"
        build.init = "sed -i -e 'g/what/ever/' /some/file"
        build.prebuild = "sed -i 'd/that wrong config/' gradle.properties"
        build.build = "./gradlew compile"
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo: chmod +rwx /opt
                init: sed -i -e 'g/what/ever/' /some/file
                prebuild: sed -i 'd/that wrong config/' gradle.properties
                build: ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_1_line_scripts_as_list(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = ["chmod +rwx /opt"]
        build.init = ["sed -i -e 'g/what/ever/' /some/file"]
        build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"]
        build.build = ["./gradlew compile"]
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo: chmod +rwx /opt
                init: sed -i -e 'g/what/ever/' /some/file
                prebuild: sed -i 'd/that wrong config/' gradle.properties
                build: ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_multiline_scripts_from_list(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = ["apt-get update",
                      "apt-get install -y whatever",
                      "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"]
        build.init = ["bash generate_some_file.sh",
                      "sed -i -e 'g/what/ever/' /some/file"]
        build.prebuild = ["npm something",
                          "echo 'important setting' >> /a/file"]
        build.build = ["./gradlew someSpecialTask",
                       "sed -i 'd/that wrong config/' gradle.properties",
                       "./gradlew compile"]
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo:
                  - apt-get update
                  - apt-get install -y whatever
                  - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                init:
                  - bash generate_some_file.sh
                  - sed -i -e 'g/what/ever/' /some/file
                prebuild:
                  - npm something
                  - echo 'important setting' >> /a/file
                build:
                  - ./gradlew someSpecialTask
                  - sed -i 'd/that wrong config/' gradle.properties
                  - ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_multiline_scripts_from_string(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = "apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"
        build.init = "bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file"
        build.prebuild = "npm something && echo 'important setting' >> /a/file"
        build.build = "./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile"
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo:
                  - apt-get update
                  - apt-get install -y whatever
                  - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                init:
                  - bash generate_some_file.sh
                  - sed -i -e 'g/what/ever/' /some/file
                prebuild:
                  - npm something
                  - echo 'important setting' >> /a/file
                build:
                  - ./gradlew someSpecialTask
                  - sed -i 'd/that wrong config/' gradle.properties
                  - ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_make_sure_provides_does_not_get_written(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.Provides = 'this.is.deprecated'
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.gradle = ['yes']
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                gradle:
                  - yes

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_parse_txt_srclib(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('JSoup.txt', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # Source details (the only mandatory fields)
                    Repo Type:git
                    Repo:https://github.com/jhy/jsoup.git

                    # Comma-separated list of subdirs to use. The first existing subdirectory
                    # found between those given will be used. If none is found or provided, the
                    # root of the repo directory will be used instead.
                    Subdir:

                    # Any extra commands to prepare the source library
                    Prepare:
                    '''))
            srclib = fdroidserver.metadata.parse_txt_srclib('JSoup.txt')
            self.assertDictEqual({'Repo': 'https://github.com/jhy/jsoup.git',
                                  'RepoType': 'git',
                                  'Subdir': [''],
                                  'Prepare': ''},
                                 srclib)

    def test_parse_txt_srclib_simple(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('simple.txt', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # this should be simple
                    Repo Type:git
                    Repo:https://git.host/repo.git
                    '''))
            srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
                                  'RepoType': 'git',
                                  'Subdir': None,
                                  'Prepare': None},
                                 srclib)

    def test_parse_txt_srclib_simple_blanks(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('simple.txt', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # this should be simple

                    Repo Type:git

                    Repo:https://git.host/repo.git

                    Subdir:

                    Prepare:
                    '''))
            srclib = fdroidserver.metadata.parse_txt_srclib('simple.txt')
            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
                                  'RepoType': 'git',
                                  'Subdir': [''],
                                  'Prepare': ''},
                                 srclib)

    def test_parse_txt_srclib_Changelog_cketti(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('Changelog-cketti.txt', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    Repo Type:git
                    Repo:https://github.com/cketti/ckChangeLog

                    Subdir:library,ckChangeLog/src/main
                    Prepare:[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\ntarget=android-19' > project.properties
                    '''))
            srclib = fdroidserver.metadata.parse_txt_srclib('Changelog-cketti.txt')
            self.assertDictEqual({'Repo': 'https://github.com/cketti/ckChangeLog',
                                  'RepoType': 'git',
                                  'Subdir': ['library', 'ckChangeLog/src/main'],
                                  'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > "
                                             "ant.properties && echo -e "
                                             "'android.library=true\\ntarget=android-19' > project.properties"},
                                 srclib)

    def test_parse_yaml_srclib_unknown_key(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('test.yml', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    RepoType: git
                    Repo: https://example.com/test.git
                    Evil: I should not be here.
                    '''))
            with self.assertRaisesRegex(MetaDataException,
                                        "Invalid srclib metadata: "
                                        "unknown key 'Evil' in "
                                        "'test.yml'"):
                fdroidserver.metadata.parse_yaml_srclib('test.yml')

    def test_parse_yaml_srclib_does_not_exists(self):
        fdroidserver.metadata.warnings_action = 'error'
        with self.assertRaisesRegex(MetaDataException,
                                    "Invalid scrlib metadata: "
                                    "'non/existent-test-srclib.yml' "
                                    "does not exist"):
            fdroidserver.metadata.parse_yaml_srclib('non/existent-test-srclib.yml')

    def test_parse_yaml_srclib_simple(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('simple.yml', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # this should be simple
                    RepoType: git
                    Repo: https://git.host/repo.git
                    '''))
            srclib = fdroidserver.metadata.parse_yaml_srclib('simple.yml')
            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
                                  'RepoType': 'git',
                                  'Subdir': None,
                                  'Prepare': None},
                                 srclib)

    def test_parse_yaml_srclib_simple_with_blanks(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('simple.yml', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # this should be simple

                    RepoType: git

                    Repo: https://git.host/repo.git

                    Subdir:

                    Prepare:
                    '''))
            srclib = fdroidserver.metadata.parse_yaml_srclib('simple.yml')
            self.assertDictEqual({'Repo': 'https://git.host/repo.git',
                                  'RepoType': 'git',
                                  'Subdir': [''],
                                  'Prepare': ''},
                                 srclib)

    def test_parse_yaml_srclib_Changelog_cketti(self):
        fdroidserver.metadata.warnings_action = 'error'
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            with open('Changelog-cketti.yml', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    RepoType: git
                    Repo: https://github.com/cketti/ckChangeLog

                    Subdir: library,ckChangeLog/src/main
                    Prepare: "[ -f project.properties ] || echo 'source.dir=java' > ant.properties && echo -e 'android.library=true\\\\ntarget=android-19' > project.properties"
                    '''))
            srclib = fdroidserver.metadata.parse_yaml_srclib('Changelog-cketti.yml')
            self.assertDictEqual(srclib,
                                 {'Repo': 'https://github.com/cketti/ckChangeLog',
                                  'RepoType': 'git',
                                  'Subdir': ['library', 'ckChangeLog/src/main'],
                                  'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > "
                                             "ant.properties && echo -e "
                                             "'android.library=true\\ntarget=android-19' > project.properties"})

    def test_read_srclibs_yml_subdir_list(self):
        fdroidserver.metadata.warnings_action = 'error'
        fdroidserver.metadata.srclibs = None
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            os.mkdir('srclibs')
            with open('srclibs/with-list.yml', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # this should be simple
                    RepoType: git
                    Repo: https://git.host/repo.git

                    Subdir:
                     - This is your last chance.
                     - After this, there is no turning back.
                     - You take the blue pill—the story ends,
                     - you wake up in your bed
                     - and believe whatever you want to believe.
                     - You take the red pill—you stay in Wonderland
                     - and I show you how deep the rabbit-hole goes.
                    Prepare:
                        There is a difference between knowing the path
                        and walking the path.
                    '''))
            fdroidserver.metadata.read_srclibs()
        self.maxDiff = None
        self.assertDictEqual(fdroidserver.metadata.srclibs,
                             {'with-list': {'RepoType': 'git',
                                            'Repo': 'https://git.host/repo.git',
                                            'Subdir': ['This is your last chance.',
                                                       'After this, there is no turning back.',
                                                       'You take the blue pill—the story ends,',
                                                       'you wake up in your bed',
                                                       'and believe whatever you want to believe.',
                                                       'You take the red pill—you stay in Wonderland',
                                                       'and I show you how deep the rabbit-hole goes.'],
                                            'Prepare': 'There is a difference between knowing the path '
                                                       'and walking the path.'}})

    def test_read_srclibs_yml_prepare_list(self):
        fdroidserver.metadata.warnings_action = 'error'
        fdroidserver.metadata.srclibs = None
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            os.mkdir('srclibs')
            with open('srclibs/with-list.yml', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # this should be simple
                    RepoType: git
                    Repo: https://git.host/repo.git

                    Subdir:
                    Prepare:
                     - The Matrix is a system, Neo.
                     - That system is our enemy.
                     - But when you're inside, you look around, what do you see?
                     - Businessmen, teachers, lawyers, carpenters.
                     - The very minds of the people we are trying to save.
                     - But until we do, these people are still a part of that system and that makes them our enemy.
                     - You have to understand, most of these people are not ready to be unplugged.
                     - And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it.
                    '''))
            fdroidserver.metadata.read_srclibs()
        self.maxDiff = None
        self.assertDictEqual(fdroidserver.metadata.srclibs,
                             {'with-list': {'RepoType': 'git',
                                            'Repo': 'https://git.host/repo.git',
                                            'Subdir': [''],
                                            'Prepare': 'The Matrix is a system, Neo. && '
                                                       'That system is our enemy. && '
                                                       'But when you\'re inside, you look around, what do you see? && '
                                                       'Businessmen, teachers, lawyers, carpenters. && '
                                                       'The very minds of the people we are trying to save. && '
                                                       'But until we do, these people are still a part of that system and that makes them our enemy. && '
                                                       'You have to understand, most of these people are not ready to be unplugged. && '
                                                       'And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it.'}})

    def test_read_srclibs(self):
        fdroidserver.metadata.warnings_action = 'error'
        fdroidserver.metadata.srclibs = None
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            os.mkdir('srclibs')
            with open('srclibs/simple.yml', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    RepoType: git
                    Repo: https://git.host/repo.git
                    '''))
            with open('srclibs/simple-wb.txt', 'w', encoding='utf-8') as f:
                f.write(textwrap.dedent('''\
                    # this should be simple
                    Repo Type:git
                    Repo:https://git.host/repo.git

                    Subdir:
                    Prepare:
                    '''))
            fdroidserver.metadata.read_srclibs()
        self.assertDictEqual(fdroidserver.metadata.srclibs,
                             {'simple-wb': {'RepoType': 'git',
                                            'Repo': 'https://git.host/repo.git',
                                            'Subdir': [''],
                                            'Prepare': ''},
                              'simple': {'RepoType': 'git',
                                         'Repo': 'https://git.host/repo.git',
                                         'Subdir': None,
                                         'Prepare': None}})

    def test_read_xref_metadata(self):
        os.chdir('xref')
        fdroidserver.metadata.warnings_action = 'error'
        apps = fdroidserver.metadata.read_metadata(xref=True)
        self.assertListEqual(list(apps.keys()),
                             ['aarddict.android', 'org.coolreader', 'org.geometerplus.zlibrary.ui.android'])


if __name__ == "__main__":
    os.chdir(os.path.dirname(__file__))

    parser = optparse.OptionParser()
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="Spew out even more information than normal")
    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])

    newSuite = unittest.TestSuite()
    newSuite.addTest(unittest.makeSuite(MetadataTest))
    unittest.main(failfast=True)