fdroidserver/tests/update.TestCase

#!/usr/bin/env python3

# http://www.drdobbs.com/testing/unit-testing-with-python/240165163

import copy
import git
import glob
import inspect
import json
import logging
import optparse
import os
import random
import shutil
import string
import subprocess
import sys
import tempfile
import unittest
import yaml
import zipfile
import textwrap
from binascii import unhexlify
from datetime import datetime
from distutils.version import LooseVersion
from testcommon import TmpCwd
from unittest import mock

try:
    from yaml import CSafeLoader as SafeLoader
except ImportError:
    from yaml import SafeLoader

try:
    from yaml import CFullLoader as FullLoader
except ImportError:
    try:
        # FullLoader is available from PyYaml 5.1+, as we don't load user
        # controlled data here, it's okay to fall back the unsafe older
        # Loader
        from yaml import FullLoader
    except ImportError:
        from yaml import Loader as FullLoader

localmodule = os.path.realpath(
    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
)
print('localmodule: ' + localmodule)
if localmodule not in sys.path:
    sys.path.insert(0, localmodule)

import fdroidserver.common
import fdroidserver.exception
import fdroidserver.metadata
import fdroidserver.update
from fdroidserver.common import FDroidPopen


DONATION_FIELDS = ('Donate', 'Liberapay', 'OpenCollective')


class Options:
    allow_disabled_algorithms = False
    clean = False
    delete_unknown = False
    nosign = False
    pretty = True
    rename_apks = False
    verbose = False


class UpdateTest(unittest.TestCase):
    '''fdroid update'''

    def setUp(self):
        logging.basicConfig(level=logging.INFO)
        logging.getLogger('androguard.axml').setLevel(logging.INFO)
        logging.getLogger('androguard.core.api_specific_resources').setLevel(logging.INFO)
        from PIL import PngImagePlugin

        logging.getLogger(PngImagePlugin.__name__).setLevel(logging.INFO)
        self.basedir = os.path.join(localmodule, 'tests')
        self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
        if not os.path.exists(self.tmpdir):
            os.makedirs(self.tmpdir)
        os.chdir(self.basedir)

        fdroidserver.common.config = None
        fdroidserver.common.options = None

    def testInsertStoreMetadata(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config
        os.chdir(os.path.join(localmodule, 'tests'))

        shutil.rmtree(os.path.join('repo', 'info.guardianproject.urzip'), ignore_errors=True)

        shutil.rmtree(os.path.join('build', 'com.nextcloud.client'), ignore_errors=True)
        shutil.copytree(os.path.join('source-files', 'com.nextcloud.client'),
                        os.path.join('build', 'com.nextcloud.client'))

        shutil.rmtree(os.path.join('build', 'com.nextcloud.client.dev'), ignore_errors=True)
        shutil.copytree(os.path.join('source-files', 'com.nextcloud.client.dev'),
                        os.path.join('build', 'com.nextcloud.client.dev'))

        shutil.rmtree(os.path.join('build', 'eu.siacs.conversations'), ignore_errors=True)
        shutil.copytree(os.path.join('source-files', 'eu.siacs.conversations'),
                        os.path.join('build', 'eu.siacs.conversations'))

        testfilename = 'icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png'
        testfile = os.path.join('repo', 'org.videolan.vlc', 'en-US', 'icon.png')
        cpdir = os.path.join('metadata', 'org.videolan.vlc', 'en-US')
        cpfile = os.path.join(cpdir, testfilename)
        os.makedirs(cpdir, exist_ok=True)
        shutil.copy(testfile, cpfile)
        shutil.copystat(testfile, cpfile)

        apps = dict()
        for packageName in (
            'info.guardianproject.urzip',
            'org.videolan.vlc',
            'obb.mainpatch.current',
            'com.nextcloud.client',
            'com.nextcloud.client.dev',
            'eu.siacs.conversations',
        ):
            apps[packageName] = fdroidserver.metadata.App()
            apps[packageName]['id'] = packageName
            apps[packageName]['CurrentVersionCode'] = 0xCAFEBEEF

        apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100

        buildnextcloudclient = fdroidserver.metadata.Build()
        buildnextcloudclient.gradle = ['generic']
        apps['com.nextcloud.client']['Builds'] = [buildnextcloudclient]

        buildnextclouddevclient = fdroidserver.metadata.Build()
        buildnextclouddevclient.gradle = ['versionDev']
        apps['com.nextcloud.client.dev']['Builds'] = [buildnextclouddevclient]

        build_conversations = fdroidserver.metadata.Build()
        build_conversations.gradle = ['free']
        apps['eu.siacs.conversations']['Builds'] = [build_conversations]

        fdroidserver.update.insert_localized_app_metadata(apps)

        appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US')
        self.assertTrue(os.path.isfile(os.path.join(
            appdir,
            'icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png')))
        self.assertTrue(os.path.isfile(os.path.join(
            appdir,
            'featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png')))

        self.assertEqual(6, len(apps))
        for packageName, app in apps.items():
            self.assertTrue('localized' in app)
            self.assertTrue('en-US' in app['localized'])
            self.assertEqual(1, len(app['localized']))
            if packageName == 'info.guardianproject.urzip':
                self.assertEqual(7, len(app['localized']['en-US']))
                self.assertEqual('full description\n', app['localized']['en-US']['description'])
                self.assertEqual('title', app['localized']['en-US']['name'])
                self.assertEqual('short description', app['localized']['en-US']['summary'])
                self.assertEqual('video', app['localized']['en-US']['video'])
                self.assertEqual('icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png',
                                 app['localized']['en-US']['icon'])
                self.assertEqual('featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png',
                                 app['localized']['en-US']['featureGraphic'])
                self.assertEqual('100\n', app['localized']['en-US']['whatsNew'])
            elif packageName == 'org.videolan.vlc':
                self.assertEqual(testfilename, app['localized']['en-US']['icon'])
                self.assertEqual(9, len(app['localized']['en-US']['phoneScreenshots']))
                self.assertEqual(15, len(app['localized']['en-US']['sevenInchScreenshots']))
            elif packageName == 'obb.mainpatch.current':
                self.assertEqual('icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png',
                                 app['localized']['en-US']['icon'])
                self.assertEqual('featureGraphic_ffhLaojxbGAfu9ROe1MJgK5ux8d0OVc6b65nmvOBaTk=.png',
                                 app['localized']['en-US']['featureGraphic'])
                self.assertEqual(1, len(app['localized']['en-US']['phoneScreenshots']))
                self.assertEqual(1, len(app['localized']['en-US']['sevenInchScreenshots']))
            elif packageName == 'com.nextcloud.client':
                self.assertEqual('Nextcloud', app['localized']['en-US']['name'])
                self.assertEqual(1073, len(app['localized']['en-US']['description']))
                self.assertEqual(78, len(app['localized']['en-US']['summary']))
            elif packageName == 'com.nextcloud.client.dev':
                self.assertEqual('Nextcloud Dev', app['localized']['en-US']['name'])
                self.assertEqual(586, len(app['localized']['en-US']['description']))
                self.assertEqual(78, len(app['localized']['en-US']['summary']))
            elif packageName == 'eu.siacs.conversations':
                self.assertEqual('Conversations', app['localized']['en-US']['name'])

    def test_name_title_scraping(self):
        """metadata file --> fdroiddata localized files --> fastlane/triple-t in app source --> APK"""
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        os.chdir(os.path.join(localmodule, 'tests'))
        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True
        fdroidserver.update.options.delete_unknown = True

        apps = fdroidserver.metadata.read_metadata()
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
        fdroidserver.update.insert_localized_app_metadata(apps)
        fdroidserver.update.apply_info_from_latest_apk(apps, apks)
        app = apps['info.guardianproject.urzip']
        self.assertIsNone(app.Name)
        self.assertTrue('localized' in app)
        self.assertEqual('title', app['localized']['en-US']['name'])
        app = apps['org.videolan.vlc']
        self.assertIsNone(app.Name)
        self.assertTrue('localized' in app)
        self.assertFalse('name' in app['localized']['en-US'])
        app = apps['info.guardianproject.checkey']
        self.assertEqual('Checkey the app!', app.Name)
        self.assertTrue('localized' in app)
        self.assertEqual('Checkey: info on local apps', app['localized']['en-US']['name'])
        self.assertEqual('Checkey: ローカルアプリの情報', app['localized']['ja-JP']['name'])
        app = apps['org.adaway']
        self.assertIsNone(app.Name)
        self.assertFalse('localized' in app)
        app = apps['obb.main.twoversions']
        self.assertIsNone(app.Name)
        self.assertFalse('localized' in app)

    def test_insert_missing_app_names_from_apks(self):
        """en-US serves as the final, default, fallback value with index-v1"""
        testvalue = 'TESTVALUE!'
        apps = {
            'none': {},
            'name': {'Name': testvalue},
            'onlyapk': {'Name': None},
            'autoname': {'AutoName': 'autoname', 'Name': None},
            'onlylocalized': {'localized': {'en-US': {'name': testvalue}}},
            'non_en_us_localized': {'localized': {'de-AT': {'name': 'leiwand'}}},
            'apks': {},
        }
        apks = [
            {'packageName': 'none', 'name': '', 'versionCode': 1},
            {'packageName': 'name', 'name': 'fromapk', 'versionCode': 1},
            {'packageName': 'onlyapk', 'name': testvalue, 'versionCode': 1},
            {'packageName': 'autoname', 'name': testvalue, 'versionCode': 1},
            {'packageName': 'onlylocalized', 'name': 'fromapk', 'versionCode': 1},
            {'packageName': 'non_en_us_localized', 'name': testvalue, 'versionCode': 0xcafe},
            {'packageName': 'apks', 'name': 'fromapk1', 'versionCode': 1},
            {'packageName': 'apks', 'name': 'fromapk2', 'versionCode': 2},
            {'packageName': 'apks', 'name': testvalue, 'versionCode': 3},
        ]
        fdroidserver.common.options = Options
        fdroidserver.update.insert_missing_app_names_from_apks(apps, apks)
        for appid, app in apps.items():
            if appid == 'none':
                self.assertIsNone(app.get('Name'))
                self.assertIsNone(app.get('localized'))
            elif appid == 'onlyapk':
                self.assertIsNone(app.get('Name'))
                self.assertEqual(testvalue, app['localized']['en-US']['name'])
            elif appid == 'autoname':
                self.assertIsNone(app.get('Name'))
                self.assertEqual(testvalue, app['localized']['en-US']['name'])
            elif appid == 'onlylocalized':
                self.assertIsNone(app.get('Name'))
                self.assertEqual(testvalue, app['localized']['en-US']['name'])
            elif appid == 'non_en_us_localized':
                self.assertIsNone(app.get('Name'))
                self.assertEqual(testvalue, app['localized']['en-US']['name'])
            elif appid == 'name':
                self.assertEqual(testvalue, app['Name'])
                self.assertIsNone(app.get('localized'))
            elif appid == 'apks':
                self.assertIsNone(app.get('Name'))
                self.assertEqual(testvalue, app['localized']['en-US']['name'])

    def test_insert_missing_app_names_from_apks_from_repo(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True
        fdroidserver.update.options.delete_unknown = True

        apps = fdroidserver.metadata.read_metadata()
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)

        appid = 'info.guardianproject.checkey'
        testapps = {appid: copy.copy(apps[appid])}
        self.assertEqual('Checkey the app!', testapps[appid]['Name'])
        del (testapps[appid]['Name'])
        fdroidserver.update.insert_missing_app_names_from_apks(testapps, apks)
        self.assertIsNone(testapps[appid].get('Name'))

        repoapps = fdroidserver.update.prepare_apps(apps, apks, 'repo')
        fdroidserver.update.insert_missing_app_names_from_apks(repoapps, apks)
        self.assertIsNone(repoapps['com.politedroid']['Name'])
        self.assertEqual('Polite Droid',
                         repoapps['com.politedroid']['localized']['en-US']['name'])
        self.assertEqual('Duplicate Permisssions', repoapps['duplicate.permisssions']['Name'])
        self.assertEqual('Caffeine Tile', repoapps['info.zwanenburg.caffeinetile']['Name'])
        self.assertEqual('No minSdkVersion or targetSdkVersion', repoapps['no.min.target.sdk']['Name'])
        self.assertIsNone(repoapps['obb.main.oldversion'].get('Name'))
        self.assertEqual('OBB Main Old Version',
                         repoapps['obb.main.oldversion']['localized']['en-US']['name'])
        self.assertIsNone(repoapps['obb.main.twoversions'].get('Name'))
        self.assertEqual('OBB Main Two Versions',
                         repoapps['obb.main.twoversions']['localized']['en-US']['name'])
        self.assertIsNone(repoapps['souch.smsbypass'].get('Name'))
        self.assertEqual('Battery level',
                         repoapps['souch.smsbypass']['localized']['en-US']['name'])
        self.assertIsNone(repoapps['info.guardianproject.urzip'].get('Name'))
        self.assertEqual('title',
                         repoapps['info.guardianproject.urzip']['localized']['en-US']['name'])
        self.assertIsNone(repoapps['obb.mainpatch.current'].get('Name'))

        del(repoapps['info.guardianproject.urzip']['localized'])
        fdroidserver.update.insert_missing_app_names_from_apks(repoapps, apks)
        self.assertEqual('urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234',
                         repoapps['info.guardianproject.urzip']['localized']['en-US']['name'])

    def test_insert_triple_t_metadata(self):
        importer = os.path.join(self.basedir, 'tmp', 'importer')
        packageName = 'org.fdroid.ci.test.app'
        if not os.path.isdir(importer):
            logging.warning('skipping test_insert_triple_t_metadata, import.TestCase must run first!')
            return
        tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name,
                                       dir=self.tmpdir)
        packageDir = os.path.join(tmptestsdir, 'build', packageName)
        shutil.copytree(importer, packageDir)

        # always use the same commit so these tests work when ci-test-app.git is updated
        repo = git.Repo(packageDir)
        for remote in repo.remotes:
            remote.fetch()
        repo.git.reset('--hard', 'b9e5d1a0d8d6fc31d4674b2f0514fef10762ed4f')
        repo.git.clean('-fdx')

        os.mkdir(os.path.join(tmptestsdir, 'metadata'))
        metadata = dict()
        metadata['Description'] = 'This is just a test app'
        with open(os.path.join(tmptestsdir, 'metadata', packageName + '.yml'), 'w') as fp:
            yaml.dump(metadata, fp)

        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        os.chdir(tmptestsdir)

        apps = fdroidserver.metadata.read_metadata()
        fdroidserver.update.copy_triple_t_store_metadata(apps)

        # TODO ideally, this would compare the whole dict like in metadata.TestCase's test_read_metadata()
        correctlocales = [
            'ar', 'ast_ES', 'az', 'ca', 'ca_ES', 'cs-CZ', 'cs_CZ', 'da',
            'da-DK', 'de', 'de-DE', 'el', 'en-US', 'es', 'es-ES', 'es_ES', 'et',
            'fi', 'fr', 'fr-FR', 'he_IL', 'hi-IN', 'hi_IN', 'hu', 'id', 'it',
            'it-IT', 'it_IT', 'iw-IL', 'ja', 'ja-JP', 'kn_IN', 'ko', 'ko-KR',
            'ko_KR', 'lt', 'nb', 'nb_NO', 'nl', 'nl-NL', 'no', 'pl', 'pl-PL',
            'pl_PL', 'pt', 'pt-BR', 'pt-PT', 'pt_BR', 'ro', 'ro_RO', 'ru-RU',
            'ru_RU', 'sv-SE', 'sv_SE', 'te', 'tr', 'tr-TR', 'uk', 'uk_UA', 'vi',
            'vi_VN', 'zh-CN', 'zh_CN', 'zh_TW',
        ]
        locales = sorted(list(apps['org.fdroid.ci.test.app']['localized'].keys()))
        self.assertEqual(correctlocales, locales)

    def test_insert_triple_t_2_metadata(self):
        packageName = 'org.piwigo.android'
        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.rmdir(tmptestsdir)
        shutil.copytree(os.path.join(self.basedir, 'triple-t-2'), tmptestsdir)
        os.chdir(tmptestsdir)

        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        apps = fdroidserver.metadata.read_metadata()
        self.assertTrue(packageName in apps)
        fdroidserver.update.copy_triple_t_store_metadata(apps)
        correctlocales = ['de-DE', 'en-US', 'fr-FR', 'kn-IN']
        app = apps[packageName]
        self.assertEqual('android@piwigo.org', app['authorEmail'])
        self.assertEqual('https://www.piwigo.org', app['authorWebSite'])
        locales = sorted(list(app['localized'].keys()))
        self.assertEqual(correctlocales, locales)
        kn_IN = app['localized']['kn-IN']
        self.assertTrue('description' in kn_IN)
        self.assertTrue('name' in kn_IN)
        self.assertTrue('summary' in kn_IN)
        en_US = app['localized']['en-US']
        self.assertTrue('whatsNew' in en_US)

        os.chdir(os.path.join('repo', packageName))
        self.assertTrue(os.path.exists(os.path.join('en-US', 'icon.png')))
        self.assertTrue(os.path.exists(os.path.join('en-US', 'featureGraphic.png')))
        self.assertTrue(os.path.exists(os.path.join('en-US', 'phoneScreenshots', '01_Login.jpg')))
        self.assertTrue(os.path.exists(os.path.join('en-US', 'sevenInchScreenshots', '01_Login.png')))
        self.assertFalse(os.path.exists(os.path.join('de-DE', 'icon.png')))
        self.assertFalse(os.path.exists(os.path.join('de-DE', 'featureGraphic.png')))
        self.assertFalse(os.path.exists(os.path.join('de-DE', 'phoneScreenshots', '01_Login.jpg')))
        self.assertFalse(os.path.exists(os.path.join('de-DE', 'sevenInchScreenshots', '01_Login.png')))

    def test_insert_triple_t_anysoftkeyboard(self):
        packages = ('com.anysoftkeyboard.languagepack.dutch', 'com.menny.android.anysoftkeyboard')
        names = ('Dutch for AnySoftKeyboard', 'AnySoftKeyboard')

        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.rmdir(tmptestsdir)
        shutil.copytree(os.path.join(self.basedir, 'triple-t-anysoftkeyboard'), tmptestsdir)
        os.chdir(tmptestsdir)

        for packageName, name in zip(packages, names):
            config = dict()
            fdroidserver.common.fill_config_defaults(config)
            fdroidserver.common.config = config
            fdroidserver.update.config = config
            fdroidserver.update.options = fdroidserver.common.options

            apps = fdroidserver.metadata.read_metadata()
            self.assertTrue(packageName in apps)
            fdroidserver.update.copy_triple_t_store_metadata(apps)
            app = apps[packageName]
            self.assertEqual(app['localized']['en-US']['name'], name)

    def test_insert_triple_t_multiple_metadata(self):
        namespace = 'ch.admin.bag.covidcertificate.'
        packages = ('verifier', 'wallet')
        names = dict(verifier='COVID Certificate Check', wallet='COVID Certificate')

        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.rmdir(tmptestsdir)
        shutil.copytree(os.path.join(self.basedir, 'triple-t-multiple'), tmptestsdir)
        os.chdir(tmptestsdir)

        for p in packages:
            packageName = namespace + p
            config = dict()
            fdroidserver.common.fill_config_defaults(config)
            fdroidserver.common.config = config
            fdroidserver.update.config = config
            fdroidserver.update.options = fdroidserver.common.options

            apps = fdroidserver.metadata.read_metadata()
            self.assertTrue(packageName in apps)
            fdroidserver.update.copy_triple_t_store_metadata(apps)
            app = apps[packageName]
            self.assertEqual(app['localized']['en-US']['name'], names[p])

    def javagetsig(self, apkfile):
        getsig_dir = 'getsig'
        if not os.path.exists(os.path.join(getsig_dir, "getsig.class")):
            logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
            sys.exit(1)
        # FDroidPopen needs some config to work
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        p = FDroidPopen(['java', '-cp', 'getsig', 'getsig', apkfile])
        sig = None
        for line in p.output.splitlines():
            if line.startswith('Result:'):
                sig = line[7:].strip()
                break
        if p.returncode == 0:
            return sig
        else:
            return None

    def testGoodGetsig(self):
        # config needed to use jarsigner and keytool
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.options = Options
        fdroidserver.update.config = config
        apkfile = 'urzip.apk'
        sig = self.javagetsig(apkfile)
        self.assertIsNotNone(sig, "sig is None")
        pysig = fdroidserver.update.getsig(apkfile)
        self.assertIsNotNone(pysig, "pysig is None")
        self.assertEqual(sig, fdroidserver.update.getsig(apkfile),
                         "python sig not equal to java sig!")
        self.assertEqual(len(sig), len(pysig),
                         "the length of the two sigs are different!")
        try:
            self.assertEqual(unhexlify(sig), unhexlify(pysig),
                             "the length of the two sigs are different!")
        except TypeError as e:
            print(e)
            self.assertTrue(False, 'TypeError!')

    def testBadGetsig(self):
        """getsig() should still be able to fetch the fingerprint of bad signatures"""
        # config needed to use jarsigner and keytool
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config

        apkfile = 'urzip-badsig.apk'
        sig = fdroidserver.update.getsig(apkfile)
        self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722',
                         "python sig should be: " + str(sig))

        apkfile = 'urzip-badcert.apk'
        sig = fdroidserver.update.getsig(apkfile)
        self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722',
                         "python sig should be: " + str(sig))

    def test_getsig(self):
        # config needed to use jarsigner and keytool
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config

        sig = fdroidserver.update.getsig('urzip-release-unsigned.apk')
        self.assertIsNone(sig)

        good_fingerprint = 'b4964fd759edaa54e65bb476d0276880'

        apkpath = 'urzip-release.apk'  # v1 only
        sig = fdroidserver.update.getsig(apkpath)
        self.assertEqual(good_fingerprint, sig,
                         'python sig was: ' + str(sig))

        apkpath = 'repo/v1.v2.sig_1020.apk'
        sig = fdroidserver.update.getsig(apkpath)
        self.assertEqual(good_fingerprint, sig,
                         'python sig was: ' + str(sig))
        # check that v1 and v2 have the same certificate
        try:
            import hashlib
            from binascii import hexlify
            from androguard.core.bytecodes.apk import APK
        except ImportError:
            print('WARNING: skipping rest of test since androguard is missing!')
            return
        apkobject = APK(apkpath)
        cert_encoded = apkobject.get_certificates_der_v2()[0]
        self.assertEqual(good_fingerprint, sig,
                         hashlib.md5(hexlify(cert_encoded)).hexdigest())  # nosec just used as ID for signing key

        filename = 'v2.only.sig_2.apk'
        with zipfile.ZipFile(filename) as z:
            self.assertTrue('META-INF/MANIFEST.MF' in z.namelist(), 'META-INF/MANIFEST.MF required')
            for f in z.namelist():
                # ensure there are no v1 signature files
                self.assertIsNone(fdroidserver.common.SIGNATURE_BLOCK_FILE_REGEX.match(f))
        sig = fdroidserver.update.getsig(filename)
        self.assertEqual(good_fingerprint, sig,
                         "python sig was: " + str(sig))

    def testScanApksAndObbs(self):
        os.chdir(os.path.join(localmodule, 'tests'))
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo')
        shutil.copytree(os.path.join(self.basedir, 'metadata'), 'metadata')
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        config['ndk_paths'] = dict()
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True
        fdroidserver.update.options.delete_unknown = True

        apps = fdroidserver.metadata.read_metadata()
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
        self.assertEqual(len(apks), 17)
        apk = apks[1]
        self.assertEqual(apk['packageName'], 'com.politedroid')
        self.assertEqual(apk['versionCode'], 3)
        self.assertEqual(apk['minSdkVersion'], 3)
        self.assertIsNone(apk.get('targetSdkVersion'))
        self.assertFalse('maxSdkVersion' in apk)
        apk = apks[8]
        self.assertEqual(apk['packageName'], 'obb.main.oldversion')
        self.assertEqual(apk['versionCode'], 1444412523)
        self.assertEqual(apk['minSdkVersion'], 4)
        self.assertEqual(apk['targetSdkVersion'], 18)
        self.assertFalse('maxSdkVersion' in apk)

        fdroidserver.update.insert_obbs('repo', apps, apks)
        for apk in apks:
            if apk['packageName'] == 'obb.mainpatch.current':
                self.assertEqual(apk.get('obbMainFile'), 'main.1619.obb.mainpatch.current.obb')
                self.assertEqual(apk.get('obbPatchFile'), 'patch.1619.obb.mainpatch.current.obb')
            elif apk['packageName'] == 'obb.main.oldversion':
                self.assertEqual(apk.get('obbMainFile'), 'main.1434483388.obb.main.oldversion.obb')
                self.assertIsNone(apk.get('obbPatchFile'))
            elif apk['packageName'] == 'obb.main.twoversions':
                self.assertIsNone(apk.get('obbPatchFile'))
                if apk['versionCode'] == 1101613:
                    self.assertEqual(apk.get('obbMainFile'), 'main.1101613.obb.main.twoversions.obb')
                elif apk['versionCode'] == 1101615:
                    self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
                elif apk['versionCode'] == 1101617:
                    self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
                else:
                    self.assertTrue(False)
            elif apk['packageName'] == 'info.guardianproject.urzip':
                self.assertIsNone(apk.get('obbMainFile'))
                self.assertIsNone(apk.get('obbPatchFile'))

    def test_apkcache_json(self):
        """test the migration from pickle to json"""
        os.chdir(os.path.join(localmodule, 'tests'))
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo')
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        config['ndk_paths'] = dict()
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True
        fdroidserver.update.options.delete_unknown = True

        fdroidserver.metadata.read_metadata()
        knownapks = fdroidserver.common.KnownApks()
        apkcache = fdroidserver.update.get_cache()
        self.assertEqual(2, len(apkcache))
        self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"])
        self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms,
                         apkcache['allow_disabled_algorithms'])
        apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False)
        fdroidserver.update.write_cache(apkcache)

        fdroidserver.update.options.clean = False
        read_from_json = fdroidserver.update.get_cache()
        self.assertEqual(19, len(read_from_json))
        for f in glob.glob('repo/*.apk'):
            self.assertTrue(os.path.basename(f) in read_from_json)

        fdroidserver.update.options.clean = True
        reset = fdroidserver.update.get_cache()
        self.assertEqual(2, len(reset))

    def test_scan_repo_files(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        os.mkdir('repo')
        os.mkdir('stats')
        with open(os.path.join('stats', 'known_apks.txt'), 'w') as fp:
            fp.write('se.manyver_30.apk se.manyver 2018-10-10\n')
        filename = 'Norway_bouvet_europe_2.obf.zip'
        shutil.copy(os.path.join(self.basedir, filename), 'repo')
        knownapks = fdroidserver.common.KnownApks()
        files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False)
        knownapks.writeifchanged()
        self.assertTrue(fcachechanged)

        info = files[0]
        self.assertEqual(filename, info['apkName'])
        self.assertEqual(datetime, type(info['added']))
        self.assertEqual(os.path.getsize(os.path.join('repo', filename)), info['size'])
        self.assertEqual(
            '531190bdbc07e77d5577249949106f32dac7f62d38d66d66c3ae058be53a729d',
            info['hash'],
        )

    def test_read_added_date_from_all_apks(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        fdroidserver.common.options = Options
        os.chdir(os.path.join(localmodule, 'tests'))
        apps = fdroidserver.metadata.read_metadata()
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
        fdroidserver.update.read_added_date_from_all_apks(apps, apks)

    def test_apply_info_from_latest_apk(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        fdroidserver.common.options = Options
        os.chdir(os.path.join(localmodule, 'tests'))
        apps = fdroidserver.metadata.read_metadata()
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
        fdroidserver.update.apply_info_from_latest_apk(apps, apks)

    def test_scan_apk(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        os.chdir(os.path.join(localmodule, 'tests'))

        if 'apksigner' in config:
            apk_info = fdroidserver.update.scan_apk('v2.only.sig_2.apk')
            self.assertIsNone(apk_info.get('maxSdkVersion'))
            self.assertEqual(apk_info.get('versionName'), 'v2-only')
            self.assertEqual(apk_info.get('versionCode'), 2)
        else:
            print('WARNING: skipping v2-only test since apksigner cannot be found')
        apk_info = fdroidserver.update.scan_apk('repo/v1.v2.sig_1020.apk')
        self.assertIsNone(apk_info.get('maxSdkVersion'))
        self.assertEqual(apk_info.get('versionName'), 'v1+2')
        self.assertEqual(apk_info.get('versionCode'), 1020)

        apk_info = fdroidserver.update.scan_apk('repo/souch.smsbypass_9.apk')
        self.assertIsNone(apk_info.get('maxSdkVersion'))
        self.assertEqual(apk_info.get('versionName'), '0.9')

        apk_info = fdroidserver.update.scan_apk('repo/duplicate.permisssions_9999999.apk')
        self.assertEqual(apk_info.get('versionName'), '')
        self.assertEqual(apk_info['icons_src'], {'160': 'res/drawable/ic_launcher.png',
                                                 '-1': 'res/drawable/ic_launcher.png'})

        apk_info = fdroidserver.update.scan_apk('org.dyndns.fules.ck_20.apk')
        self.assertEqual(apk_info['icons_src'], {'240': 'res/drawable-hdpi-v4/icon_launcher.png',
                                                 '120': 'res/drawable-ldpi-v4/icon_launcher.png',
                                                 '160': 'res/drawable-mdpi-v4/icon_launcher.png',
                                                 '-1': 'res/drawable-mdpi-v4/icon_launcher.png'})
        self.assertEqual(apk_info['icons'], {})
        self.assertEqual(apk_info['features'], [])
        self.assertEqual(apk_info['antiFeatures'], set())
        self.assertEqual(apk_info['versionName'], 'v1.6pre2')
        self.assertEqual(apk_info['hash'],
                         '897486e1f857c6c0ee32ccbad0e1b8cd82f6d0e65a44a23f13f852d2b63a18c8')
        self.assertEqual(apk_info['packageName'], 'org.dyndns.fules.ck')
        self.assertEqual(apk_info['versionCode'], 20)
        self.assertEqual(apk_info['size'], 132453)
        self.assertEqual(apk_info['nativecode'],
                         ['arm64-v8a', 'armeabi', 'armeabi-v7a', 'mips', 'mips64', 'x86', 'x86_64'])
        self.assertEqual(apk_info['minSdkVersion'], 7)
        self.assertEqual(apk_info['sig'], '9bf7a6a67f95688daec75eab4b1436ac')
        self.assertEqual(apk_info['hashType'], 'sha256')
        self.assertEqual(apk_info['targetSdkVersion'], 8)

        apk_info = fdroidserver.update.scan_apk('org.bitbucket.tickytacky.mirrormirror_4.apk')
        self.assertEqual(apk_info.get('versionName'), '1.0.3')
        self.assertEqual(apk_info['icons_src'], {'160': 'res/drawable-mdpi/mirror.png',
                                                 '-1': 'res/drawable-mdpi/mirror.png'})

        apk_info = fdroidserver.update.scan_apk('repo/info.zwanenburg.caffeinetile_4.apk')
        self.assertEqual(apk_info.get('versionName'), '1.3')
        self.assertEqual(apk_info['icons_src'], {})

        apk_info = fdroidserver.update.scan_apk('repo/com.politedroid_6.apk')
        self.assertEqual(apk_info.get('versionName'), '1.5')
        self.assertEqual(apk_info['icons_src'], {'120': 'res/drawable-ldpi-v4/icon.png',
                                                 '160': 'res/drawable-mdpi-v4/icon.png',
                                                 '240': 'res/drawable-hdpi-v4/icon.png',
                                                 '320': 'res/drawable-xhdpi-v4/icon.png',
                                                 '-1': 'res/drawable-mdpi-v4/icon.png'})

        apk_info = fdroidserver.update.scan_apk('SpeedoMeterApp.main_1.apk')
        self.assertEqual(apk_info.get('versionName'), '1.0')
        self.assertEqual(apk_info['icons_src'], {})

    def test_scan_apk_no_min_target(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        apk_info = fdroidserver.update.scan_apk('repo/no.min.target.sdk_987.apk')
        self.maxDiff = None
        self.assertDictEqual(apk_info, {
            'icons': {},
            'icons_src': {'-1': 'res/drawable/ic_launcher.png',
                          '160': 'res/drawable/ic_launcher.png'},
            'name': 'No minSdkVersion or targetSdkVersion',
            'signer': '32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6',
            'hashType': 'sha256',
            'packageName': 'no.min.target.sdk',
            'features': [],
            'antiFeatures': set(),
            'size': 14102,
            'sig': 'b4964fd759edaa54e65bb476d0276880',
            'versionName': '1.2-fake',
            'uses-permission-sdk-23': [],
            'hash': 'e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c',
            'versionCode': 987,
            'minSdkVersion': 3,
            'uses-permission': [
                fdroidserver.update.UsesPermission(name='android.permission.WRITE_EXTERNAL_STORAGE',
                                                   maxSdkVersion=None),
                fdroidserver.update.UsesPermission(name='android.permission.READ_PHONE_STATE',
                                                   maxSdkVersion=None),
                fdroidserver.update.UsesPermission(name='android.permission.READ_EXTERNAL_STORAGE',
                                                   maxSdkVersion=None)]})

    def test_scan_apk_no_sig(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config
        os.chdir(os.path.join(localmodule, 'tests'))
        if os.path.basename(os.getcwd()) != 'tests':
            raise Exception('This test must be run in the "tests/" subdir')

        with self.assertRaises(fdroidserver.exception.BuildException):
            fdroidserver.update.scan_apk('urzip-release-unsigned.apk')

    def test_scan_apk_bad_zip(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        os.mkdir('repo')
        apkfile = 'repo/badzip_1.apk'
        with open(apkfile, 'w') as fp:
            fp.write('this is not a zip file')
        with self.assertRaises(fdroidserver.exception.BuildException):
            fdroidserver.update.scan_apk(apkfile)

    @unittest.skipUnless(
        os.path.exists('tests/SystemWebView-repack.apk'), "file too big for sdist"
    )
    def test_scan_apk_bad_icon_id(self):
        """Some APKs can produce an exception when extracting the icon

        This kind of parsing exception should be reported then ignored
        so that working APKs can be included in the index.  There are
        so many weird things that make it into APKs, that does not
        automatically disqualify them from inclusion. For example:

        ValueError: invalid literal for int() with base 16: '<0x801FF, type 0x07>'

        The test APK was made from:
        https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1018#note_690565333
        It was then stripped down by doing:

        * mkdir SystemWebView
        * cd SystemWebView/
        * unzip ../SystemWebView.apk
        * rm -rf META-INF/ lib assets/icudtl.dat assets/stored-locales/
        * jar cf ../SystemWebView-repack.apk *
        """
        # reset the state, perhaps this should be in setUp()
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            os.mkdir('repo')
            apkfile = 'repo/SystemWebView-repack.apk'
            shutil.copy(os.path.join(self.basedir, os.path.basename(apkfile)), apkfile)
            fdroidserver.update.scan_apk(apkfile)

    def test_process_apk(self):
        def _build_yaml_representer(dumper, data):
            '''Creates a YAML representation of a Build instance'''
            return dumper.represent_dict(data)

        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config
        os.chdir(os.path.join(localmodule, 'tests'))

        config['ndk_paths'] = dict()
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True
        fdroidserver.update.options.delete_unknown = True

        for icon_dir in fdroidserver.update.get_all_icon_dirs('repo'):
            if not os.path.exists(icon_dir):
                os.makedirs(icon_dir)

        knownapks = fdroidserver.common.KnownApks()
        apkList = ['../urzip.apk', '../org.dyndns.fules.ck_20.apk']

        for apkName in apkList:
            _, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks,
                                                                   False)
            # Don't care about the date added to the repo and relative apkName
            self.assertEqual(datetime, type(apk['added']))
            del apk['added']
            del apk['apkName']

            # ensure that icons have been extracted properly
            if apkName == '../urzip.apk':
                self.assertEqual(apk['icon'], 'info.guardianproject.urzip.100.png')
            if apkName == '../org.dyndns.fules.ck_20.apk':
                self.assertEqual(apk['icon'], 'org.dyndns.fules.ck.20.png')
            for density in fdroidserver.update.screen_densities:
                icon_path = os.path.join(
                    fdroidserver.update.get_icon_dir('repo', density), apk['icon']
                )
                self.assertTrue(os.path.isfile(icon_path))
                self.assertTrue(os.path.getsize(icon_path) > 1)

            savepath = os.path.join('metadata', 'apk', apk['packageName'] + '.yaml')
            # Uncomment to save APK metadata
            # with open(savepath, 'w') as f:
            #     yaml.add_representer(fdroidserver.metadata.Build, _build_yaml_representer)
            #     yaml.dump(apk, f, default_flow_style=False)

            # CFullLoader doesn't always work
            # https://github.com/yaml/pyyaml/issues/266#issuecomment-559116876
            TestLoader = FullLoader
            try:
                testyaml = '- !!python/object/new:fdroidserver.update.UsesPermission\n  - test\n  - null'
                from_yaml = yaml.load(testyaml, Loader=TestLoader)
            except yaml.constructor.ConstructorError:
                from yaml import UnsafeLoader as TestLoader

            with open(savepath, 'r') as f:
                from_yaml = yaml.load(f, Loader=TestLoader)
            self.maxDiff = None
            self.assertEqual(apk, from_yaml)

    def test_process_apk_signed_by_disabled_algorithms(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config

        config['ndk_paths'] = dict()
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True
        fdroidserver.update.options.verbose = True
        fdroidserver.update.options.delete_unknown = True

        knownapks = fdroidserver.common.KnownApks()

        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        print('tmptestsdir', tmptestsdir)
        os.chdir(tmptestsdir)
        os.mkdir('repo')
        os.mkdir('archive')
        # setup the repo, create icons dirs, etc.
        fdroidserver.update.process_apks({}, 'repo', knownapks)
        fdroidserver.update.process_apks({}, 'archive', knownapks)

        disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk']
        for apkName in disabledsigs:
            shutil.copy(os.path.join(self.basedir, apkName),
                        os.path.join(tmptestsdir, 'repo'))

            skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
                                                                      knownapks,
                                                                      allow_disabled_algorithms=True,
                                                                      archive_bad_sig=False)
            self.assertFalse(skip)
            self.assertIsNotNone(apk)
            self.assertTrue(cachechanged)
            self.assertFalse(os.path.exists(os.path.join('archive', apkName)))
            self.assertTrue(os.path.exists(os.path.join('repo', apkName)))

            if os.path.exists('/usr/bin/apksigner') or 'apksigner' in config:
                print('SKIPPING: apksigner installed and it allows MD5 signatures')
                return

            javac = config['jarsigner'].replace('jarsigner', 'javac')
            v = subprocess.check_output([javac, '-version'], stderr=subprocess.STDOUT)[6:-1].decode('utf-8')
            if LooseVersion(v) < LooseVersion('1.8.0_132'):
                print('SKIPPING: running tests with old Java (' + v + ')')
                return

            # this test only works on systems with fully updated Java/jarsigner
            # that has MD5 listed in jdk.jar.disabledAlgorithms in java.security
            # https://blogs.oracle.com/java-platform-group/oracle-jre-will-no-longer-trust-md5-signed-code-by-default
            skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
                                                                      knownapks,
                                                                      allow_disabled_algorithms=False,
                                                                      archive_bad_sig=True)
            self.assertTrue(skip)
            self.assertIsNone(apk)
            self.assertFalse(cachechanged)
            self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
            self.assertFalse(os.path.exists(os.path.join('repo', apkName)))

            skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'archive',
                                                                      knownapks,
                                                                      allow_disabled_algorithms=False,
                                                                      archive_bad_sig=False)
            self.assertFalse(skip)
            self.assertIsNotNone(apk)
            self.assertTrue(cachechanged)
            self.assertTrue(os.path.exists(os.path.join('archive', apkName)))
            self.assertFalse(os.path.exists(os.path.join('repo', apkName)))

            # ensure that icons have been moved to the archive as well
            for density in fdroidserver.update.screen_densities:
                icon_path = os.path.join(fdroidserver.update.get_icon_dir('archive', density),
                                         apk['icon'])
                self.assertTrue(os.path.isfile(icon_path))
                self.assertTrue(os.path.getsize(icon_path) > 1)

        badsigs = ['urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release-unsigned.apk', ]
        for apkName in badsigs:
            shutil.copy(os.path.join(self.basedir, apkName),
                        os.path.join(tmptestsdir, 'repo'))

            skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo',
                                                                      knownapks,
                                                                      allow_disabled_algorithms=False,
                                                                      archive_bad_sig=False)
            self.assertTrue(skip)
            self.assertIsNone(apk)
            self.assertFalse(cachechanged)

    def test_process_invalid_apk(self):
        os.chdir(os.path.join(localmodule, 'tests'))
        if os.path.basename(os.getcwd()) != 'tests':
            raise Exception('This test must be run in the "tests/" subdir')

        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config
        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.delete_unknown = False

        knownapks = fdroidserver.common.KnownApks()
        apk = 'fake.ota.update_1234.zip'  # this is not an APK, scanning should fail
        (skip, apk, cachechanged) = fdroidserver.update.process_apk({}, apk, 'repo', knownapks,
                                                                    False)

        self.assertTrue(skip)
        self.assertIsNone(apk)
        self.assertFalse(cachechanged)

    def test_get_apks_without_allowed_signatures(self):
        """Test when no AllowedAPKSigningKeys is specified"""
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.common.options = Options

        app = fdroidserver.metadata.App()
        knownapks = fdroidserver.common.KnownApks()
        apkfile = 'v1.v2.sig_1020.apk'
        (skip, apk, cachechanged) = fdroidserver.update.process_apk(
            {}, apkfile, 'repo', knownapks, False
        )

        r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk)
        self.assertIsNone(r)

    def test_get_apks_without_allowed_signatures_allowed(self):
        """Test when the APK matches the specified AllowedAPKSigningKeys"""
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options

        app = fdroidserver.metadata.App(
            {
                'AllowedAPKSigningKeys': '32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6'
            }
        )
        knownapks = fdroidserver.common.KnownApks()
        apkfile = 'v1.v2.sig_1020.apk'
        (skip, apk, cachechanged) = fdroidserver.update.process_apk(
            {}, apkfile, 'repo', knownapks, False
        )

        r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk)
        self.assertIsNone(r)

    def test_get_apks_without_allowed_signatures_blocked(self):
        """Test when the APK does not match any specified AllowedAPKSigningKeys"""
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options

        app = fdroidserver.metadata.App(
            {
                'AllowedAPKSigningKeys': 'fa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edead'
            }
        )
        knownapks = fdroidserver.common.KnownApks()
        apkfile = 'v1.v2.sig_1020.apk'
        (skip, apk, cachechanged) = fdroidserver.update.process_apk(
            {}, apkfile, 'repo', knownapks, False
        )

        r = fdroidserver.update.get_apks_without_allowed_signatures(app, apk)
        self.assertEqual(apkfile, r)

    def test_update_with_AllowedAPKSigningKeys(self):
        """Test that APKs without allowed signatures get deleted."""
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        os.mkdir('repo')
        testapk = os.path.join('repo', 'com.politedroid_6.apk')
        shutil.copy(os.path.join(self.basedir, testapk), testapk)
        os.mkdir('metadata')
        metadatafile = os.path.join('metadata', 'com.politedroid.yml')
        shutil.copy(os.path.join(self.basedir, metadatafile), metadatafile)
        with open(metadatafile, 'a') as fp:
            fp.write(
                '\n\nAllowedAPKSigningKeys: 32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6\n'
            )

        fdroidserver.common.options = Options
        config = fdroidserver.common.read_config(fdroidserver.common.options)
        config['repo_keyalias'] = 'sova'
        config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
        config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
        config['keystore'] = os.path.join(self.basedir, 'keystore.jks')

        self.assertTrue(os.path.exists(testapk))
        with mock.patch('sys.argv', ['fdroid update', '--delete-unknown']):
            fdroidserver.update.main()
        self.assertTrue(os.path.exists(testapk))

        with open(metadatafile, 'a') as fp:
            fp.write(
                '\n\nAllowedAPKSigningKeys: fa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edeadfa4edead\n'
            )
        with mock.patch('sys.argv', ['fdroid update', '--delete-unknown']):
            fdroidserver.update.main()
        self.assertFalse(os.path.exists(testapk))

    def test_translate_per_build_anti_features(self):
        os.chdir(os.path.join(localmodule, 'tests'))
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo')
        shutil.copytree(os.path.join(self.basedir, 'metadata'), 'metadata')
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        config['ndk_paths'] = dict()
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True
        fdroidserver.update.options.delete_unknown = True

        apps = fdroidserver.metadata.read_metadata()
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
        fdroidserver.update.translate_per_build_anti_features(apps, apks)
        self.assertEqual(len(apks), 17)
        foundtest = False
        for apk in apks:
            if apk['packageName'] == 'com.politedroid' and apk['versionCode'] == 3:
                antiFeatures = apk.get('antiFeatures')
                self.assertTrue('KnownVuln' in antiFeatures)
                self.assertEqual(3, len(antiFeatures))
                foundtest = True
        self.assertTrue(foundtest)

    def test_create_metadata_from_template(self):
        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        print('tmptestsdir', tmptestsdir)
        os.chdir(tmptestsdir)
        os.mkdir('repo')
        os.mkdir('metadata')
        shutil.copy(os.path.join(localmodule, 'tests', 'urzip.apk'), 'repo')

        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        config['ndk_paths'] = dict()
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        fdroidserver.common.options = Options
        fdroidserver.update.options = fdroidserver.common.options
        fdroidserver.update.options.clean = True

        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
        self.assertEqual(1, len(apks))
        apk = apks[0]

        testfile = 'metadata/info.guardianproject.urzip.yml'
        # create empty 0 byte .yml file, run read_metadata, it should work
        open(testfile, 'a').close()
        apps = fdroidserver.metadata.read_metadata()
        self.assertEqual(1, len(apps))
        os.remove(testfile)

        # test using internal template
        apps = fdroidserver.metadata.read_metadata()
        self.assertEqual(0, len(apps))
        fdroidserver.update.create_metadata_from_template(apk)
        self.assertTrue(os.path.exists(testfile))
        apps = fdroidserver.metadata.read_metadata()
        self.assertEqual(1, len(apps))
        for app in apps.values():
            self.assertEqual('urzip', app['Name'])
            self.assertEqual(1, len(app['Categories']))
            break

        # test using external template.yml
        os.remove(testfile)
        self.assertFalse(os.path.exists(testfile))
        shutil.copy(os.path.join(localmodule, 'examples', 'template.yml'), tmptestsdir)
        fdroidserver.update.create_metadata_from_template(apk)
        self.assertTrue(os.path.exists(testfile))
        apps = fdroidserver.metadata.read_metadata()
        self.assertEqual(1, len(apps))
        for app in apps.values():
            self.assertEqual('urzip', app['Name'])
            self.assertEqual(1, len(app['Categories']))
            self.assertEqual('Internet', app['Categories'][0])
            break
        with open(testfile) as fp:
            data = yaml.load(fp, Loader=SafeLoader)
        self.assertEqual('urzip', data['Name'])
        self.assertEqual('urzip', data['Summary'])

    def test_has_known_vulnerability(self):
        good = [
            'org.bitbucket.tickytacky.mirrormirror_1.apk',
            'org.bitbucket.tickytacky.mirrormirror_2.apk',
            'org.bitbucket.tickytacky.mirrormirror_3.apk',
            'org.bitbucket.tickytacky.mirrormirror_4.apk',
            'org.dyndns.fules.ck_20.apk',
            'urzip.apk',
            'urzip-badcert.apk',
            'urzip-badsig.apk',
            'urzip-release.apk',
            'urzip-release-unsigned.apk',
            'repo/com.politedroid_3.apk',
            'repo/com.politedroid_4.apk',
            'repo/com.politedroid_5.apk',
            'repo/com.politedroid_6.apk',
            'repo/obb.main.oldversion_1444412523.apk',
            'repo/obb.mainpatch.current_1619_another-release-key.apk',
            'repo/obb.mainpatch.current_1619.apk',
            'repo/obb.main.twoversions_1101613.apk',
            'repo/obb.main.twoversions_1101615.apk',
            'repo/obb.main.twoversions_1101617.apk',
            'repo/urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk',
        ]
        for f in good:
            self.assertFalse(fdroidserver.update.has_known_vulnerability(f))
        with self.assertRaises(fdroidserver.exception.FDroidException):
            fdroidserver.update.has_known_vulnerability('janus.apk')

    def test_get_apk_icon_when_src_is_none(self):
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.common.config = config
        fdroidserver.update.config = config

        # pylint: disable=protected-access
        icons_src = fdroidserver.update._get_apk_icons_src('urzip-release.apk', None)
        assert icons_src == {}

    def test_strip_and_copy_image(self):
        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )

        in_file = os.path.join(self.basedir, 'metadata', 'info.guardianproject.urzip', 'en-US', 'images', 'icon.png')
        out_file = os.path.join(tmptestsdir, 'icon.png')
        fdroidserver.update._strip_and_copy_image(in_file, out_file)
        self.assertTrue(os.path.exists(out_file))

        in_file = os.path.join(self.basedir, 'corrupt-featureGraphic.png')
        out_file = os.path.join(tmptestsdir, 'corrupt-featureGraphic.png')
        fdroidserver.update._strip_and_copy_image(in_file, out_file)
        self.assertFalse(os.path.exists(out_file))

    def test_create_metadata_from_template_empty_keys(self):
        apk = {'packageName': 'rocks.janicerand'}
        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
            os.mkdir('metadata')
            with open('template.yml', 'w') as f:
                f.write(
                    textwrap.dedent(
                        '''\
                    Disabled:
                    License:
                    AuthorName:
                    AuthorEmail:
                    AuthorWebSite:
                    WebSite:
                    SourceCode:
                    IssueTracker:
                    Translation:
                    Changelog:
                    Donate:
                    FlattrID:
                    LiberapayID:
                    Bitcoin:
                    Litecoin:
                    Name:
                    AutoName:
                    Summary:
                    RequiresRoot:
                    RepoType:
                    Repo:
                    Binaries:
                    Builds:
                    ArchivePolicy:
                    AutoUpdateMode:
                    UpdateCheckMode:
                    UpdateCheckIgnore:
                    VercodeOperation:
                    UpdateCheckName:
                    UpdateCheckData:
                    CurrentVersion:
                    CurrentVersionCode:
                    NoSourceSince:
                    '''
                    )
                )
            fdroidserver.update.create_metadata_from_template(apk)
            with open(os.path.join('metadata', 'rocks.janicerand.yml')) as f:
                metadata_content = yaml.load(f, Loader=SafeLoader)
                self.maxDiff = None
                self.assertDictEqual(
                    metadata_content,
                    {
                        'ArchivePolicy': '',
                        'AuthorEmail': '',
                        'AuthorName': '',
                        'AuthorWebSite': '',
                        'AutoName': 'rocks.janicerand',
                        'AutoUpdateMode': '',
                        'Binaries': '',
                        'Bitcoin': '',
                        'Builds': '',
                        'Changelog': '',
                        'CurrentVersion': '',
                        'CurrentVersionCode': '',
                        'Disabled': '',
                        'Donate': '',
                        'FlattrID': '',
                        'IssueTracker': '',
                        'LiberapayID': '',
                        'License': '',
                        'Litecoin': '',
                        'Name': 'rocks.janicerand',
                        'NoSourceSince': '',
                        'Repo': '',
                        'RepoType': '',
                        'RequiresRoot': '',
                        'SourceCode': '',
                        'Summary': 'rocks.janicerand',
                        'Translation': '',
                        'UpdateCheckData': '',
                        'UpdateCheckIgnore': '',
                        'UpdateCheckMode': '',
                        'UpdateCheckName': '',
                        'VercodeOperation': '',
                        'WebSite': '',
                    },
                )

    def test_insert_funding_yml_donation_links(self):
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        os.mkdir('build')
        content = textwrap.dedent(
            """
            community_bridge: ''
            custom: [LINK1, LINK2]
            github: USERNAME
            issuehunt: USERNAME
            ko_fi: USERNAME
            liberapay: USERNAME
            open_collective: USERNAME
            otechie: USERNAME
            patreon: USERNAME
        """
        )
        app = fdroidserver.metadata.App()
        app.id = 'fake.app.id'
        apps = {app.id: app}
        os.mkdir(os.path.join('build', app.id))
        fdroidserver.update.insert_funding_yml_donation_links(apps)
        for field in DONATION_FIELDS:
            self.assertFalse(app.get(field))
        with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
            fp.write(content)

        fdroidserver.update.insert_funding_yml_donation_links(apps)
        for field in DONATION_FIELDS:
            self.assertIsNotNone(app.get(field), field)
        self.assertEqual('LINK1', app.get('Donate'))
        self.assertEqual('USERNAME', app.get('Liberapay'))
        self.assertEqual('USERNAME', app.get('OpenCollective'))

        app['Donate'] = 'keepme'
        app['Liberapay'] = 'keepme'
        app['OpenCollective'] = 'keepme'
        fdroidserver.update.insert_funding_yml_donation_links(apps)
        for field in DONATION_FIELDS:
            self.assertEqual('keepme', app.get(field))

    def test_insert_funding_yml_donation_links_one_at_a_time(self):
        """Exercise the FUNDING.yml code one entry at a time"""
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        os.mkdir('build')

        app = fdroidserver.metadata.App()
        app.id = 'fake.app.id'
        apps = {app.id: app}
        os.mkdir(os.path.join('build', app.id))
        fdroidserver.update.insert_funding_yml_donation_links(apps)
        for field in DONATION_FIELDS:
            self.assertIsNone(app.get(field))

        content = textwrap.dedent(
            """
            community_bridge: 'blah-de-blah'
            github: USERNAME
            issuehunt: USERNAME
            ko_fi: USERNAME
            liberapay: USERNAME
            open_collective: USERNAME
            patreon: USERNAME
        """
        )
        for line in content.split('\n'):
            if not line:
                continue
            app = fdroidserver.metadata.App()
            app.id = 'fake.app.id'
            apps = {app.id: app}
            with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
                fp.write(line)
            data = yaml.load(line, Loader=SafeLoader)
            fdroidserver.update.insert_funding_yml_donation_links(apps)
            if 'liberapay' in data:
                self.assertEqual(data['liberapay'], app.get('Liberapay'))
            elif 'open_collective' in data:
                self.assertEqual(data['open_collective'], app.get('OpenCollective'))
            else:
                for v in data.values():
                    self.assertEqual(app.get('Donate', '').split('/')[-1], v)

    def test_insert_funding_yml_donation_links_with_corrupt_file(self):
        testdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(testdir)
        os.mkdir('build')
        app = fdroidserver.metadata.App()
        app.id = 'fake.app.id'
        apps = {app.id: app}
        os.mkdir(os.path.join('build', app.id))
        with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
            fp.write(
                textwrap.dedent(
                    """
                opencollective: foo
                custom: []
                liberapay: :
            """
                )
            )
        fdroidserver.update.insert_funding_yml_donation_links(apps)
        for field in DONATION_FIELDS:
            self.assertIsNone(app.get(field))

    def test_sanitize_funding_yml(self):
        with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
            data = yaml.load(fp, Loader=SafeLoader)
        for k, entries in data.items():
            for entry in entries:
                if k in 'custom':
                    m = fdroidserver.update.sanitize_funding_yml_entry(entry)
                else:
                    m = fdroidserver.update.sanitize_funding_yml_name(entry)
                if k == 'bad':
                    self.assertIsNone(m)
                else:
                    self.assertIsNotNone(m)
        self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry('foo\nbar'))
        self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry(
            ''.join(chr(random.randint(65, 90)) for _ in range(2049))))

        # not recommended but valid entries
        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(12345))
        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(5.0))
        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(' WhyIncludeWhitespace '))
        self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(['first', 'second']))

    def test_set_localized_text_entry(self):
        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(tmptestsdir)
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config
        fdroidserver.update.options = fdroidserver.common.options

        files = {
            'full-description.txt': 'description',
            'short-description.txt': 'summary',
            'title.txt': 'name',
            'video-url.txt': 'video',
        }

        for f, key in files.items():
            limit = config['char_limits'][key]
            with open(f, 'w') as fp:
                fp.write(''.join(random.choice(string.ascii_letters) for i in range(limit + 100)))
            locale = 'ru_US'
            app = dict()
            fdroidserver.update._set_localized_text_entry(app, locale, key, f)
            self.assertEqual(limit, len(app['localized'][locale][key]))

            f = 'badlink-' + f
            os.symlink('/path/to/nowhere', f)
            app = dict()
            fdroidserver.update._set_localized_text_entry(app, locale, key, f)
            self.assertIsNone(app['localized'].get(locale, {}).get(key))

    def test_set_author_entry(self):
        tmptestsdir = tempfile.mkdtemp(
            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
        )
        os.chdir(tmptestsdir)
        config = dict()
        fdroidserver.common.fill_config_defaults(config)
        fdroidserver.update.config = config
        fdroidserver.update.options = fdroidserver.common.options

        f = 'contact-website.txt'
        key = 'author'
        url = 'https://f-droid.org/'
        limit = config['char_limits']['author']
        with open(f, 'w') as fp:
            fp.write(url)
            fp.write('\n')
        app = dict()
        fdroidserver.update._set_author_entry(app, key, f)
        self.assertEqual(url, app[key])

        f = 'limits.txt'
        key = 'author'
        limit = config['char_limits']['author']
        for key in ('authorEmail', 'authorPhone', 'authorWebSite'):
            with open(f, 'w') as fp:
                fp.write(''.join(random.choice(string.ascii_letters) for i in range(limit + 100)))
            app = dict()
            fdroidserver.update._set_author_entry(app, key, f)
            self.assertEqual(limit, len(app[key]))

        f = 'badlink.txt'
        os.symlink('/path/to/nowhere', f)
        app = dict()
        fdroidserver.update._set_author_entry(app, key, f)
        self.assertIsNone(app.get(key))

    def test_status_update_json(self):
        fdroidserver.common.config = {}
        fdroidserver.update.config = {}
        fdroidserver.update.options = Options
        with tempfile.TemporaryDirectory() as tmpdir:
            os.chdir(tmpdir)
            with mock.patch('sys.argv', ['fdroid update', '']):
                fdroidserver.update.status_update_json([], [])
                with open('repo/status/update.json') as fp:
                    data = json.load(fp)
                self.assertTrue('apksigner' in data)

                fdroidserver.update.config = {
                    'apksigner': 'apksigner',
                }
                fdroidserver.update.status_update_json([], [])
                with open('repo/status/update.json') as fp:
                    data = json.load(fp)
                self.assertEqual(shutil.which(fdroidserver.update.config['apksigner']), data['apksigner'])

                fdroidserver.update.config = {}
                fdroidserver.common.fill_config_defaults(fdroidserver.update.config)
                fdroidserver.update.status_update_json([], [])
                with open('repo/status/update.json') as fp:
                    data = json.load(fp)
                self.assertEqual(fdroidserver.update.config.get('apksigner'), data['apksigner'])
                self.assertEqual(fdroidserver.update.config['jarsigner'], data['jarsigner'])
                self.assertEqual(fdroidserver.update.config['keytool'], data['keytool'])


if __name__ == "__main__":
    os.chdir(os.path.dirname(__file__))

    parser = optparse.OptionParser()
    parser.add_option(
        "-v",
        "--verbose",
        action="store_true",
        default=False,
        help="Spew out even more information than normal",
    )
    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])

    newSuite = unittest.TestSuite()
    newSuite.addTest(unittest.makeSuite(UpdateTest))
    unittest.main(failfast=False)