fdroidserver/tests/metadata.TestCase

#!/usr/bin/env python3

# http://www.drdobbs.com/testing/unit-testing-with-python/240165163

import io
import glob
import inspect
import logging
import optparse
import os
import random
import shutil
import sys
import unittest
import yaml
import tempfile
import textwrap
from unittest import mock

localmodule = os.path.realpath(
    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
print('localmodule: ' + localmodule)
if localmodule not in sys.path:
    sys.path.insert(0, localmodule)

import fdroidserver.common
import fdroidserver.metadata
from fdroidserver.exception import MetaDataException


class MetadataTest(unittest.TestCase):
    '''fdroidserver/metadata.py'''

    def setUp(self):
        logging.basicConfig(level=logging.DEBUG)
        self.basedir = os.path.join(localmodule, 'tests')
        self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
        if not os.path.exists(self.tmpdir):
            os.makedirs(self.tmpdir)
        os.chdir(self.basedir)

    def test_FieldValidator_BitcoinAddress(self):
        validator = None
        for vali in fdroidserver.metadata.valuetypes:
            if vali.name == 'Bitcoin address':
                validator = vali
                break
        self.assertIsNotNone(validator, "could not find 'Bitcoin address' validator")

        fdroidserver.metadata.warnings_action = 'error'

        # some valid addresses (P2PKH, P2SH, Bech32)
        self.assertIsNone(validator.check('1BrrrrErsrWetrTrnrrrrm4GFg7xJaNVN2', 'fake.app.id'))
        self.assertIsNone(validator.check('3JrrrrWrEZr3rNrrvrecrnyirrnqRhWNLy', 'fake.app.id'))
        self.assertIsNone(validator.check('bc1qar0srrr7xrkvr5lr43lrdnwrre5rgtrzrf5rrq', 'fake.app.id'))

        # some invalid addresses
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '21BvMrSYsrWrtrrlL5A10mlGFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '5Hrgr3ur5rGLrfKrrrrrrHSrqJrroGrrzrQrrrrrrLNrsrDrrrA', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'L1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')

    def test_FieldValidator_LitecoinAddress(self):
        validator = None
        for vali in fdroidserver.metadata.valuetypes:
            if vali.name == 'Litecoin address':
                validator = vali
                break
        self.assertIsNotNone(validator, "could not find 'Litecoin address' validator")

        fdroidserver.metadata.warnings_action = 'error'

        # some valid addresses (L, M, 3)
        self.assertIsNone(validator.check('LgeGrrrrJAxyXprrPrrBrrX5Qrrrrrrrrd', 'fake.app.id'))
        self.assertIsNone(validator.check('MrrrrrrrJAxyXpanPtrrRAX5QHxvUJo8id', 'fake.app.id'))
        self.assertIsNone(validator.check('3rereVr9rAryrranrrrrrAXrrHx', 'fake.app.id'))

        # some invalid addresses (various special use/testnet addresses, invalid chars)
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '21BvMrSYsrWrtrrrn5Au4l4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '5Hrgr3ur5rGLrfKrrrrrr1SrqJrroGrrzrQrrrrrrLNrsrDrrrA', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          '92rr46rUrgTrrromrVrirW6r1rrrdrerrdbJrrrhrCsYrrrrrrc', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'K1BvMrSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'L0000rSYsrWrtrrrn5Au4m4GFr7rrarrN2', 'fake.app.id')
        self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
                          'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')

    def test_read_metadata(self):

        def _build_yaml_representer(dumper, data):
            '''Creates a YAML representation of a Build instance'''
            return dumper.represent_dict(data)

        self.maxDiff = None

        # these need to be set to prevent code running on None, only
        # 'accepted_formats' is actually used in metadata.py
        config = dict()
        config['sdk_path'] = '/opt/android-sdk'
        config['ndk_paths'] = dict()
        config['accepted_formats'] = ['json', 'txt', 'yml']
        fdroidserver.common.config = config
        fdroidserver.metadata.warnings_action = None

        apps = fdroidserver.metadata.read_metadata(xref=True)
        for appid in ('org.smssecure.smssecure', 'org.adaway',
                      'org.videolan.vlc', 'com.politedroid'):
            savepath = os.path.join('metadata', 'dump', appid + '.yaml')
            frommeta = dict(apps[appid])
            self.assertTrue(appid in apps)
            with open(savepath, 'r') as f:
                frompickle = yaml.load(f)
            self.assertEqual(frommeta, frompickle)
            # comment above assert and uncomment below to update test
            # files when new metadata fields are added
            # with open(savepath, 'w') as f:
            #     yaml.add_representer(fdroidserver.metadata.Build, _build_yaml_representer)
            #     yaml.dump(frommeta, f, default_flow_style=False)

    def test_rewrite_yaml_fakeotaupdate(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']}
        fdroidserver.metadata.warnings_action = None

        # rewrite metadata
        allapps = fdroidserver.metadata.read_metadata(xref=True)
        for appid, app in allapps.items():
            if appid == 'fake.ota.update':
                fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)

        # assert rewrite result
        with open(os.path.join(testdir, 'fake.ota.update.yml'), 'r') as result:
            with open('metadata-rewrite-yml/fake.ota.update.yml', 'r') as orig:
                self.maxDiff = None
                self.assertEqual(result.read(), orig.read())

    def test_rewrite_yaml_fdroidclient(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']}

        # rewrite metadata
        allapps = fdroidserver.metadata.read_metadata(xref=True)
        for appid, app in allapps.items():
            if appid == 'org.fdroid.fdroid':
                fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)

        # assert rewrite result
        with open(os.path.join(testdir, 'org.fdroid.fdroid.yml'), 'r') as result:
            with open('metadata-rewrite-yml/org.fdroid.fdroid.yml', 'r') as orig:
                self.maxDiff = None
                self.assertEqual(result.read(), orig.read())

    def test_rewrite_yaml_special_build_params(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']}

        # rewrite metadata
        allapps = fdroidserver.metadata.read_metadata(xref=True)
        for appid, app in allapps.items():
            if appid == 'app.with.special.build.params':
                fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)

        # assert rewrite result
        with open(os.path.join(testdir, 'app.with.special.build.params.yml'), 'r') as result:
            with open('metadata-rewrite-yml/app.with.special.build.params.yml', 'r') as orig:
                self.maxDiff = None
                self.assertEqual(result.read(), orig.read())

    def test_read_metadata_sort_by_time(self):
        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
        metadatadir = os.path.join(testdir, 'metadata')
        os.makedirs(metadatadir)
        fdroidserver.common.config = {'accepted_formats': ['txt']}

        randomlist = []
        randomapps = glob.glob(os.path.join(self.basedir, 'metadata', '*.txt'))
        random.shuffle(randomapps)
        i = 1
        for f in randomapps:
            shutil.copy(f, metadatadir)
            new = os.path.join(metadatadir, os.path.basename(f))
            stat = os.stat(new)
            os.utime(new, (stat.st_ctime, stat.st_mtime + i))
            # prepend new item so newest is always first
            randomlist = [os.path.basename(f)[:-4]] + randomlist
            i += 1
        os.chdir(testdir)
        allapps = fdroidserver.metadata.read_metadata(xref=True, sort_by_time=True)
        allappids = []
        for appid, app in allapps.items():
            allappids.append(appid)
        self.assertEqual(randomlist, allappids)

    def test_parse_yaml_metadata_unknown_app_field(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds: []
            bad: value"""))
        mf.name = 'mock_filename.yaml'
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            with self.assertRaises(MetaDataException):
                fdroidserver.metadata.parse_yaml_metadata(mf, {})

    def test_parse_yaml_metadata_unknown_build_flag(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
              - bad: value"""))
        mf.name = 'mock_filename.yaml'
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            with self.assertRaises(MetaDataException):
                fdroidserver.metadata.parse_yaml_metadata(mf, {})

    def test_write_yaml_with_placeholder_values(self):
        mf = io.StringIO()

        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.SourceCode = "https://gitlab.com/fdroid/fdroidclient.git"
        app.IssueTracker = "https://gitlab.com/fdroid/fdroidclient/issues"
        app.RepoType = 'git'
        app.Repo = 'https://gitlab.com/fdroid/fdroidclient.git'
        app.AutoUpdateMode = 'None'
        app.UpdateCheckMode = 'Tags'
        build = fdroidserver.metadata.Build()
        build.versionName = 'Unknown'  # taken from fdroidserver/import.py
        build.versionCode = '0'  # taken from fdroidserver/import.py
        build.disable = 'Generated by import.py ...'
        build.commit = 'Unknown'
        build.gradle = [True]
        app.builds = [build]

        fdroidserver.metadata.write_yaml(mf, app)

        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown
            SourceCode: https://gitlab.com/fdroid/fdroidclient.git
            IssueTracker: https://gitlab.com/fdroid/fdroidclient/issues

            RepoType: git
            Repo: https://gitlab.com/fdroid/fdroidclient.git

            Builds:
              - versionName: Unknown
                versionCode: 0
                disable: Generated by import.py ...
                commit: Unknown
                gradle:
                  - true

            AutoUpdateMode: None
            UpdateCheckMode: Tags
            """))

    def test_parse_yaml_metadata_prebuild_list(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  sudo:
                    - apt-get update
                    - apt-get install -y whatever
                    - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                  init:
                    - bash generate_some_file.sh
                    - sed -i -e 'g/what/ever/' /some/file
                  prebuild:
                    - npm something
                    - echo 'important setting' >> /a/file
                  build:
                    - ./gradlew someSpecialTask
                    - sed -i 'd/that wrong config/' gradle.properties
                    - ./gradlew compile
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
        self.maxDiff = None
        self.assertDictEqual(result, {'AutoName': 'F-Droid',
                                      'RepoType': 'git',
                                      'Builds': [{'versionCode': 1,
                                                  'versionName': 'v0.1.0',
                                                  'sudo': "apt-get update && "
                                                          "apt-get install -y whatever && "
                                                          "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
                                                  'init': "bash generate_some_file.sh && "
                                                          "sed -i -e 'g/what/ever/' /some/file",
                                                  'prebuild': "npm something && echo 'important setting' >> /a/file",
                                                  'build': "./gradlew someSpecialTask && "
                                                           "sed -i 'd/that wrong config/' gradle.properties && "
                                                           "./gradlew compile"}]})

    def test_parse_yaml_metadata_prebuild_strings(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  sudo: |-
                    apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                  init: bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file
                  prebuild: npm something && echo 'important setting' >> /a/file
                  build: |-
                    ./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
        self.maxDiff = None
        self.assertDictEqual(result, {'AutoName': 'F-Droid',
                                      'RepoType': 'git',
                                      'Builds': [{'versionCode': 1,
                                                  'versionName': 'v0.1.0',
                                                  'sudo': "apt-get update && "
                                                          "apt-get install -y whatever && "
                                                          "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
                                                  'init': "bash generate_some_file.sh && "
                                                          "sed -i -e 'g/what/ever/' /some/file",
                                                  'prebuild': "npm something && echo 'important setting' >> /a/file",
                                                  'build': "./gradlew someSpecialTask && "
                                                           "sed -i 'd/that wrong config/' gradle.properties && "
                                                           "./gradlew compile"}]})

    def test_parse_yaml_metadata_prebuild_string(self):
        mf = io.StringIO(textwrap.dedent("""\
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  prebuild: |-
                    a && b && sed -i 's,a,b,'
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
        self.assertDictEqual(result, {'AutoName': 'F-Droid',
                                      'RepoType': 'git',
                                      'Builds': [{'versionCode': 1,
                                                  'versionName': 'v0.1.0',
                                                  'prebuild': "a && b && "
                                                              "sed -i 's,a,b,'"}]})

    def test_parse_yaml_provides_should_be_ignored(self):
        mf = io.StringIO(textwrap.dedent("""\
            Provides: this.is.deprecated
            AutoName: F-Droid
            RepoType: git
            Builds:
                - versionCode: 1
                  versionName: v0.1.0
                  prebuild: |-
                    a && b && sed -i 's,a,b,'
            """))
        mf.name = 'mock_filename.yaml'
        mf.seek(0)
        result = {}
        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
            fdroidserver.metadata.parse_yaml_metadata(mf, result)
            self.assertNotIn('Provides', result)
            self.assertNotIn('provides', result)

    def test_write_yaml_1_line_scripts_as_string(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = "chmod +rwx /opt"
        build.init = "sed -i -e 'g/what/ever/' /some/file"
        build.prebuild = "sed -i 'd/that wrong config/' gradle.properties"
        build.build = "./gradlew compile"
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo: chmod +rwx /opt
                init: sed -i -e 'g/what/ever/' /some/file
                prebuild: sed -i 'd/that wrong config/' gradle.properties
                build: ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_1_line_scripts_as_list(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = ["chmod +rwx /opt"]
        build.init = ["sed -i -e 'g/what/ever/' /some/file"]
        build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"]
        build.build = ["./gradlew compile"]
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo: chmod +rwx /opt
                init: sed -i -e 'g/what/ever/' /some/file
                prebuild: sed -i 'd/that wrong config/' gradle.properties
                build: ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_multiline_scripts_from_list(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = ["apt-get update",
                      "apt-get install -y whatever",
                      "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"]
        build.init = ["bash generate_some_file.sh",
                      "sed -i -e 'g/what/ever/' /some/file"]
        build.prebuild = ["npm something",
                          "echo 'important setting' >> /a/file"]
        build.build = ["./gradlew someSpecialTask",
                       "sed -i 'd/that wrong config/' gradle.properties",
                       "./gradlew compile"]
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo:
                  - apt-get update
                  - apt-get install -y whatever
                  - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                init:
                  - bash generate_some_file.sh
                  - sed -i -e 'g/what/ever/' /some/file
                prebuild:
                  - npm something
                  - echo 'important setting' >> /a/file
                build:
                  - ./gradlew someSpecialTask
                  - sed -i 'd/that wrong config/' gradle.properties
                  - ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_multiline_scripts_from_string(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.sudo = "apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"
        build.init = "bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file"
        build.prebuild = "npm something && echo 'important setting' >> /a/file"
        build.build = "./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile"
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                sudo:
                  - apt-get update
                  - apt-get install -y whatever
                  - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
                init:
                  - bash generate_some_file.sh
                  - sed -i -e 'g/what/ever/' /some/file
                prebuild:
                  - npm something
                  - echo 'important setting' >> /a/file
                build:
                  - ./gradlew someSpecialTask
                  - sed -i 'd/that wrong config/' gradle.properties
                  - ./gradlew compile

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))

    def test_write_yaml_make_sure_provides_does_not_get_written(self):
        mf = io.StringIO()
        app = fdroidserver.metadata.App()
        app.Categories = ['None']
        app.Provides = 'this.is.deprecated'
        app.builds = []
        build = fdroidserver.metadata.Build()
        build.versionCode = 102030
        build.versionName = 'v1.2.3'
        build.gradle = ['yes']
        app.builds.append(build)
        fdroidserver.metadata.write_yaml(mf, app)
        mf.seek(0)
        self.assertEqual(mf.read(), textwrap.dedent("""\
            Categories:
              - None
            License: Unknown

            Builds:
              - versionName: v1.2.3
                versionCode: 102030
                gradle:
                  - yes

            AutoUpdateMode: None
            UpdateCheckMode: None
            """))


if __name__ == "__main__":
    os.chdir(os.path.dirname(__file__))

    parser = optparse.OptionParser()
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="Spew out even more information than normal")
    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])

    newSuite = unittest.TestSuite()
    newSuite.addTest(unittest.makeSuite(MetadataTest))
    unittest.main(failfast=True)