#!/usr/bin/env python3 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 import git import glob import inspect import logging import optparse import os import random import shutil import subprocess import sys import tempfile import unittest import yaml import zipfile import textwrap from binascii import unhexlify from datetime import datetime from distutils.version import LooseVersion from testcommon import TmpCwd try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader try: from yaml import CFullLoader as FullLoader except ImportError: try: # FullLoader is available from PyYaml 5.1+, as we don't load user # controlled data here, it's okay to fall back the unsafe older # Loader from yaml import FullLoader except ImportError: from yaml import Loader as FullLoader localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) print('localmodule: ' + localmodule) if localmodule not in sys.path: sys.path.insert(0, localmodule) import fdroidserver.common import fdroidserver.exception import fdroidserver.metadata import fdroidserver.update from fdroidserver.common import FDroidPopen DONATION_FIELDS = ( 'Donate', 'Liberapay', 'OpenCollective', ) class Options: allow_disabled_algorithms = False clean = False rename_apks = False class UpdateTest(unittest.TestCase): '''fdroid update''' def setUp(self): logging.basicConfig(level=logging.INFO) self.basedir = os.path.join(localmodule, 'tests') self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) os.chdir(self.basedir) def testInsertStoreMetadata(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config fdroidserver.update.options = fdroidserver.common.options os.chdir(os.path.join(localmodule, 'tests')) shutil.rmtree(os.path.join('repo', 'info.guardianproject.urzip'), ignore_errors=True) shutil.rmtree(os.path.join('build', 'com.nextcloud.client'), ignore_errors=True) shutil.copytree(os.path.join('source-files', 'com.nextcloud.client'), os.path.join('build', 'com.nextcloud.client')) shutil.rmtree(os.path.join('build', 'com.nextcloud.client.dev'), ignore_errors=True) shutil.copytree(os.path.join('source-files', 'com.nextcloud.client.dev'), os.path.join('build', 'com.nextcloud.client.dev')) shutil.rmtree(os.path.join('build', 'eu.siacs.conversations'), ignore_errors=True) shutil.copytree(os.path.join('source-files', 'eu.siacs.conversations'), os.path.join('build', 'eu.siacs.conversations')) testfilename = 'icon_yAfSvPRJukZzMMfUzvbYqwaD1XmHXNtiPBtuPVHW-6s=.png' testfile = os.path.join('repo', 'org.videolan.vlc', 'en-US', 'icon.png') cpdir = os.path.join('metadata', 'org.videolan.vlc', 'en-US') cpfile = os.path.join(cpdir, testfilename) os.makedirs(cpdir, exist_ok=True) shutil.copy(testfile, cpfile) shutil.copystat(testfile, cpfile) apps = dict() for packageName in ('info.guardianproject.urzip', 'org.videolan.vlc', 'obb.mainpatch.current', 'com.nextcloud.client', 'com.nextcloud.client.dev', 'eu.siacs.conversations'): apps[packageName] = fdroidserver.metadata.App() apps[packageName]['id'] = packageName apps[packageName]['CurrentVersionCode'] = 0xcafebeef apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100 buildnextcloudclient = fdroidserver.metadata.Build() buildnextcloudclient.gradle = ['generic'] apps['com.nextcloud.client']['builds'] = [buildnextcloudclient] buildnextclouddevclient = fdroidserver.metadata.Build() buildnextclouddevclient.gradle = ['versionDev'] apps['com.nextcloud.client.dev']['builds'] = [buildnextclouddevclient] build_conversations = fdroidserver.metadata.Build() build_conversations.gradle = ['free'] apps['eu.siacs.conversations']['builds'] = [build_conversations] fdroidserver.update.insert_localized_app_metadata(apps) appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US') self.assertTrue(os.path.isfile(os.path.join( appdir, 'icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png'))) self.assertTrue(os.path.isfile(os.path.join( appdir, 'featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png'))) self.assertEqual(6, len(apps)) for packageName, app in apps.items(): self.assertTrue('localized' in app) self.assertTrue('en-US' in app['localized']) self.assertEqual(1, len(app['localized'])) if packageName == 'info.guardianproject.urzip': self.assertEqual(7, len(app['localized']['en-US'])) self.assertEqual('full description\n', app['localized']['en-US']['description']) self.assertEqual('title', app['localized']['en-US']['name']) self.assertEqual('short description', app['localized']['en-US']['summary']) self.assertEqual('video', app['localized']['en-US']['video']) self.assertEqual('icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png', app['localized']['en-US']['icon']) self.assertEqual('featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png', app['localized']['en-US']['featureGraphic']) self.assertEqual('100\n', app['localized']['en-US']['whatsNew']) elif packageName == 'org.videolan.vlc': self.assertEqual(testfilename, app['localized']['en-US']['icon']) self.assertEqual(9, len(app['localized']['en-US']['phoneScreenshots'])) self.assertEqual(15, len(app['localized']['en-US']['sevenInchScreenshots'])) elif packageName == 'obb.mainpatch.current': self.assertEqual('icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png', app['localized']['en-US']['icon']) self.assertEqual('featureGraphic_ffhLaojxbGAfu9ROe1MJgK5ux8d0OVc6b65nmvOBaTk=.png', app['localized']['en-US']['featureGraphic']) self.assertEqual(1, len(app['localized']['en-US']['phoneScreenshots'])) self.assertEqual(1, len(app['localized']['en-US']['sevenInchScreenshots'])) elif packageName == 'com.nextcloud.client': self.assertEqual('Nextcloud', app['localized']['en-US']['name']) self.assertEqual(1073, len(app['localized']['en-US']['description'])) self.assertEqual(78, len(app['localized']['en-US']['summary'])) elif packageName == 'com.nextcloud.client.dev': self.assertEqual('Nextcloud Dev', app['localized']['en-US']['name']) self.assertEqual(586, len(app['localized']['en-US']['description'])) self.assertEqual(78, len(app['localized']['en-US']['summary'])) elif packageName == 'eu.siacs.conversations': self.assertEqual('Conversations', app['localized']['en-US']['name']) def test_insert_triple_t_metadata(self): importer = os.path.join(self.basedir, 'tmp', 'importer') packageName = 'org.fdroid.ci.test.app' if not os.path.isdir(importer): logging.warning('skipping test_insert_triple_t_metadata, import.TestCase must run first!') return tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) packageDir = os.path.join(tmptestsdir, 'build', packageName) shutil.copytree(importer, packageDir) # always use the same commit so these tests work when ci-test-app.git is updated repo = git.Repo(packageDir) for remote in repo.remotes: remote.fetch() repo.git.reset('--hard', 'b9e5d1a0d8d6fc31d4674b2f0514fef10762ed4f') repo.git.clean('-fdx') os.mkdir(os.path.join(tmptestsdir, 'metadata')) metadata = dict() metadata['Description'] = 'This is just a test app' with open(os.path.join(tmptestsdir, 'metadata', packageName + '.yml'), 'w') as fp: yaml.dump(metadata, fp) config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = fdroidserver.common.options os.chdir(tmptestsdir) apps = fdroidserver.metadata.read_metadata() fdroidserver.update.copy_triple_t_store_metadata(apps) # TODO ideally, this would compare the whole dict like in metadata.TestCase's test_read_metadata() correctlocales = [ 'ar', 'ast_ES', 'az', 'ca', 'ca_ES', 'cs-CZ', 'cs_CZ', 'da', 'da-DK', 'de', 'de-DE', 'el', 'en-US', 'es', 'es-ES', 'es_ES', 'et', 'fi', 'fr', 'fr-FR', 'he_IL', 'hi-IN', 'hi_IN', 'hu', 'id', 'it', 'it-IT', 'it_IT', 'iw-IL', 'ja', 'ja-JP', 'kn_IN', 'ko', 'ko-KR', 'ko_KR', 'lt', 'nb', 'nb_NO', 'nl', 'nl-NL', 'no', 'pl', 'pl-PL', 'pl_PL', 'pt', 'pt-BR', 'pt-PT', 'pt_BR', 'ro', 'ro_RO', 'ru-RU', 'ru_RU', 'sv-SE', 'sv_SE', 'te', 'tr', 'tr-TR', 'uk', 'uk_UA', 'vi', 'vi_VN', 'zh-CN', 'zh_CN', 'zh_TW', ] locales = sorted(list(apps['org.fdroid.ci.test.app']['localized'].keys())) self.assertEqual(correctlocales, locales) def test_insert_triple_t_2_metadata(self): packageName = 'org.piwigo.android' tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.rmdir(tmptestsdir) shutil.copytree(os.path.join(self.basedir, 'triple-t-2'), tmptestsdir) os.chdir(tmptestsdir) config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = fdroidserver.common.options apps = fdroidserver.metadata.read_metadata() self.assertTrue(packageName in apps) fdroidserver.update.copy_triple_t_store_metadata(apps) correctlocales = ['de-DE', 'en-US', 'fr-FR', 'kn-IN'] app = apps[packageName] self.assertEqual('android@piwigo.org', app['authorEmail']) self.assertEqual('https://www.piwigo.org', app['authorWebSite']) locales = sorted(list(app['localized'].keys())) self.assertEqual(correctlocales, locales) kn_IN = app['localized']['kn-IN'] self.assertTrue('description' in kn_IN) self.assertTrue('name' in kn_IN) self.assertTrue('summary' in kn_IN) en_US = app['localized']['en-US'] self.assertTrue('whatsNew' in en_US) os.chdir(os.path.join('repo', packageName)) self.assertTrue(os.path.exists(os.path.join('en-US', 'icon.png'))) self.assertTrue(os.path.exists(os.path.join('en-US', 'featureGraphic.png'))) self.assertTrue(os.path.exists(os.path.join('en-US', 'phoneScreenshots', '01_Login.jpg'))) self.assertTrue(os.path.exists(os.path.join('en-US', 'sevenInchScreenshots', '01_Login.png'))) self.assertFalse(os.path.exists(os.path.join('de-DE', 'icon.png'))) self.assertFalse(os.path.exists(os.path.join('de-DE', 'featureGraphic.png'))) self.assertFalse(os.path.exists(os.path.join('de-DE', 'phoneScreenshots', '01_Login.jpg'))) self.assertFalse(os.path.exists(os.path.join('de-DE', 'sevenInchScreenshots', '01_Login.png'))) def javagetsig(self, apkfile): getsig_dir = 'getsig' if not os.path.exists(getsig_dir + "/getsig.class"): logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir) sys.exit(1) # FDroidPopen needs some config to work config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config p = FDroidPopen(['java', '-cp', 'getsig', 'getsig', apkfile]) sig = None for line in p.output.splitlines(): if line.startswith('Result:'): sig = line[7:].strip() break if p.returncode == 0: return sig else: return None def testGoodGetsig(self): # config needed to use jarsigner and keytool config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config apkfile = 'urzip.apk' sig = self.javagetsig(apkfile) self.assertIsNotNone(sig, "sig is None") pysig = fdroidserver.update.getsig(apkfile) self.assertIsNotNone(pysig, "pysig is None") self.assertEqual(sig, fdroidserver.update.getsig(apkfile), "python sig not equal to java sig!") self.assertEqual(len(sig), len(pysig), "the length of the two sigs are different!") try: self.assertEqual(unhexlify(sig), unhexlify(pysig), "the length of the two sigs are different!") except TypeError as e: print(e) self.assertTrue(False, 'TypeError!') def testBadGetsig(self): """getsig() should still be able to fetch the fingerprint of bad signatures""" # config needed to use jarsigner and keytool config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config apkfile = 'urzip-badsig.apk' sig = fdroidserver.update.getsig(apkfile) self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722', "python sig should be: " + str(sig)) apkfile = 'urzip-badcert.apk' sig = fdroidserver.update.getsig(apkfile) self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722', "python sig should be: " + str(sig)) def test_getsig(self): # config needed to use jarsigner and keytool config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config sig = fdroidserver.update.getsig('urzip-release-unsigned.apk') self.assertIsNone(sig) good_fingerprint = 'b4964fd759edaa54e65bb476d0276880' apkpath = 'urzip-release.apk' # v1 only sig = fdroidserver.update.getsig(apkpath) self.assertEqual(good_fingerprint, sig, 'python sig was: ' + str(sig)) apkpath = 'repo/v1.v2.sig_1020.apk' sig = fdroidserver.update.getsig(apkpath) self.assertEqual(good_fingerprint, sig, 'python sig was: ' + str(sig)) # check that v1 and v2 have the same certificate try: import hashlib from binascii import hexlify from androguard.core.bytecodes.apk import APK except ImportError: print('WARNING: skipping rest of test since androguard is missing!') return apkobject = APK(apkpath) cert_encoded = apkobject.get_certificates_der_v2()[0] self.assertEqual(good_fingerprint, sig, hashlib.md5(hexlify(cert_encoded)).hexdigest()) # nosec just used as ID for signing key filename = 'v2.only.sig_2.apk' with zipfile.ZipFile(filename) as z: self.assertTrue('META-INF/MANIFEST.MF' in z.namelist(), 'META-INF/MANIFEST.MF required') for f in z.namelist(): # ensure there are no v1 signature files self.assertIsNone(fdroidserver.common.SIGNATURE_BLOCK_FILE_REGEX.match(f)) sig = fdroidserver.update.getsig(filename) self.assertEqual(good_fingerprint, sig, "python sig was: " + str(sig)) def testScanApksAndObbs(self): os.chdir(os.path.join(localmodule, 'tests')) testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo') shutil.copytree(os.path.join(self.basedir, 'metadata'), 'metadata') config = dict() fdroidserver.common.fill_config_defaults(config) config['ndk_paths'] = dict() fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.delete_unknown = True fdroidserver.update.options.rename_apks = False fdroidserver.update.options.allow_disabled_algorithms = False apps = fdroidserver.metadata.read_metadata() knownapks = fdroidserver.common.KnownApks() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) self.assertEqual(len(apks), 17) apk = apks[1] self.assertEqual(apk['packageName'], 'com.politedroid') self.assertEqual(apk['versionCode'], 3) self.assertEqual(apk['minSdkVersion'], 3) self.assertIsNone(apk.get('targetSdkVersion')) self.assertFalse('maxSdkVersion' in apk) apk = apks[8] self.assertEqual(apk['packageName'], 'obb.main.oldversion') self.assertEqual(apk['versionCode'], 1444412523) self.assertEqual(apk['minSdkVersion'], 4) self.assertEqual(apk['targetSdkVersion'], 18) self.assertFalse('maxSdkVersion' in apk) fdroidserver.update.insert_obbs('repo', apps, apks) for apk in apks: if apk['packageName'] == 'obb.mainpatch.current': self.assertEqual(apk.get('obbMainFile'), 'main.1619.obb.mainpatch.current.obb') self.assertEqual(apk.get('obbPatchFile'), 'patch.1619.obb.mainpatch.current.obb') elif apk['packageName'] == 'obb.main.oldversion': self.assertEqual(apk.get('obbMainFile'), 'main.1434483388.obb.main.oldversion.obb') self.assertIsNone(apk.get('obbPatchFile')) elif apk['packageName'] == 'obb.main.twoversions': self.assertIsNone(apk.get('obbPatchFile')) if apk['versionCode'] == 1101613: self.assertEqual(apk.get('obbMainFile'), 'main.1101613.obb.main.twoversions.obb') elif apk['versionCode'] == 1101615: self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb') elif apk['versionCode'] == 1101617: self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb') else: self.assertTrue(False) elif apk['packageName'] == 'info.guardianproject.urzip': self.assertIsNone(apk.get('obbMainFile')) self.assertIsNone(apk.get('obbPatchFile')) def test_apkcache_json(self): """test the migration from pickle to json""" os.chdir(os.path.join(localmodule, 'tests')) testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo') config = dict() fdroidserver.common.fill_config_defaults(config) config['ndk_paths'] = dict() fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.delete_unknown = True fdroidserver.update.options.rename_apks = False fdroidserver.update.options.allow_disabled_algorithms = False fdroidserver.metadata.read_metadata() knownapks = fdroidserver.common.KnownApks() apkcache = fdroidserver.update.get_cache() self.assertEqual(2, len(apkcache)) self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"]) self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms, apkcache['allow_disabled_algorithms']) apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False) fdroidserver.update.write_cache(apkcache) fdroidserver.update.options.clean = False read_from_json = fdroidserver.update.get_cache() self.assertEqual(19, len(read_from_json)) for f in glob.glob('repo/*.apk'): self.assertTrue(os.path.basename(f) in read_from_json) fdroidserver.update.options.clean = True reset = fdroidserver.update.get_cache() self.assertEqual(2, len(reset)) def test_scan_repo_files(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) os.mkdir('repo') os.mkdir('stats') with open(os.path.join('stats', 'known_apks.txt'), 'w') as fp: fp.write('se.manyver_30.apk se.manyver 2018-10-10\n') filename = 'Norway_bouvet_europe_2.obf.zip' shutil.copy(os.path.join(self.basedir, filename), 'repo') knownapks = fdroidserver.common.KnownApks() files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False) knownapks.writeifchanged() self.assertTrue(fcachechanged) info = files[0] self.assertEqual(filename, info['apkName']) self.assertEqual(datetime, type(info['added'])) self.assertEqual(os.path.getsize(os.path.join('repo', filename)), info['size']) self.assertEqual('531190bdbc07e77d5577249949106f32dac7f62d38d66d66c3ae058be53a729d', info['hash']) def test_read_added_date_from_all_apks(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = Options os.chdir(os.path.join(localmodule, 'tests')) apps = fdroidserver.metadata.read_metadata() knownapks = fdroidserver.common.KnownApks() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks) fdroidserver.update.read_added_date_from_all_apks(apps, apks) def test_apply_info_from_latest_apk(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = Options os.chdir(os.path.join(localmodule, 'tests')) apps = fdroidserver.metadata.read_metadata() knownapks = fdroidserver.common.KnownApks() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks) fdroidserver.update.apply_info_from_latest_apk(apps, apks) def test_scan_apk(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) apksigner = fdroidserver.common.find_apksigner() if apksigner: config['apksigner'] = apksigner apk_info = fdroidserver.update.scan_apk('v2.only.sig_2.apk') self.assertIsNone(apk_info.get('maxSdkVersion')) self.assertEqual(apk_info.get('versionName'), 'v2-only') self.assertEqual(apk_info.get('versionCode'), 2) else: print('WARNING: skipping v2-only test since apksigner cannot be found') apk_info = fdroidserver.update.scan_apk('repo/v1.v2.sig_1020.apk') self.assertIsNone(apk_info.get('maxSdkVersion')) self.assertEqual(apk_info.get('versionName'), 'v1+2') self.assertEqual(apk_info.get('versionCode'), 1020) apk_info = fdroidserver.update.scan_apk('repo/souch.smsbypass_9.apk') self.assertIsNone(apk_info.get('maxSdkVersion')) self.assertEqual(apk_info.get('versionName'), '0.9') apk_info = fdroidserver.update.scan_apk('repo/duplicate.permisssions_9999999.apk') self.assertEqual(apk_info.get('versionName'), '') self.assertEqual(apk_info['icons_src'], {'160': 'res/drawable/ic_launcher.png', '-1': 'res/drawable/ic_launcher.png'}) apk_info = fdroidserver.update.scan_apk('org.dyndns.fules.ck_20.apk') self.assertEqual(apk_info['icons_src'], {'240': 'res/drawable-hdpi-v4/icon_launcher.png', '120': 'res/drawable-ldpi-v4/icon_launcher.png', '160': 'res/drawable-mdpi-v4/icon_launcher.png', '-1': 'res/drawable-mdpi-v4/icon_launcher.png'}) self.assertEqual(apk_info['icons'], {}) self.assertEqual(apk_info['features'], []) self.assertEqual(apk_info['antiFeatures'], set()) self.assertEqual(apk_info['versionName'], 'v1.6pre2') self.assertEqual(apk_info['hash'], '897486e1f857c6c0ee32ccbad0e1b8cd82f6d0e65a44a23f13f852d2b63a18c8') self.assertEqual(apk_info['packageName'], 'org.dyndns.fules.ck') self.assertEqual(apk_info['versionCode'], 20) self.assertEqual(apk_info['size'], 132453) self.assertEqual(apk_info['nativecode'], ['arm64-v8a', 'armeabi', 'armeabi-v7a', 'mips', 'mips64', 'x86', 'x86_64']) self.assertEqual(apk_info['minSdkVersion'], 7) self.assertEqual(apk_info['sig'], '9bf7a6a67f95688daec75eab4b1436ac') self.assertEqual(apk_info['hashType'], 'sha256') self.assertEqual(apk_info['targetSdkVersion'], 8) apk_info = fdroidserver.update.scan_apk('org.bitbucket.tickytacky.mirrormirror_4.apk') self.assertEqual(apk_info.get('versionName'), '1.0.3') self.assertEqual(apk_info['icons_src'], {'160': 'res/drawable-mdpi/mirror.png', '-1': 'res/drawable-mdpi/mirror.png'}) apk_info = fdroidserver.update.scan_apk('repo/info.zwanenburg.caffeinetile_4.apk') self.assertEqual(apk_info.get('versionName'), '1.3') self.assertEqual(apk_info['icons_src'], {'160': 'res/drawable/ic_coffee_on.xml', '-1': 'res/drawable/ic_coffee_on.xml'}) apk_info = fdroidserver.update.scan_apk('repo/com.politedroid_6.apk') self.assertEqual(apk_info.get('versionName'), '1.5') self.assertEqual(apk_info['icons_src'], {'120': 'res/drawable-ldpi-v4/icon.png', '160': 'res/drawable-mdpi-v4/icon.png', '240': 'res/drawable-hdpi-v4/icon.png', '320': 'res/drawable-xhdpi-v4/icon.png', '-1': 'res/drawable-mdpi-v4/icon.png'}) apk_info = fdroidserver.update.scan_apk('SpeedoMeterApp.main_1.apk') self.assertEqual(apk_info.get('versionName'), '1.0') self.assertEqual(apk_info['icons_src'], {}) def test_scan_apk_no_min_target(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config apk_info = fdroidserver.update.scan_apk('repo/no.min.target.sdk_987.apk') self.maxDiff = None self.assertDictEqual(apk_info, { 'icons': {}, 'icons_src': {'-1': 'res/drawable/ic_launcher.png', '160': 'res/drawable/ic_launcher.png'}, 'name': 'No minSdkVersion or targetSdkVersion', 'signer': '32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6', 'hashType': 'sha256', 'packageName': 'no.min.target.sdk', 'features': [], 'antiFeatures': set(), 'size': 14102, 'sig': 'b4964fd759edaa54e65bb476d0276880', 'versionName': '1.2-fake', 'uses-permission-sdk-23': [], 'hash': 'e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c', 'versionCode': 987, 'minSdkVersion': 3, 'uses-permission': [ fdroidserver.update.UsesPermission(name='android.permission.WRITE_EXTERNAL_STORAGE', maxSdkVersion=None), fdroidserver.update.UsesPermission(name='android.permission.READ_PHONE_STATE', maxSdkVersion=None), fdroidserver.update.UsesPermission(name='android.permission.READ_EXTERNAL_STORAGE', maxSdkVersion=None)]}) def test_scan_apk_no_sig(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) if os.path.basename(os.getcwd()) != 'tests': raise Exception('This test must be run in the "tests/" subdir') with self.assertRaises(fdroidserver.exception.BuildException): fdroidserver.update.scan_apk('urzip-release-unsigned.apk') def test_scan_apk_bad_zip(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) os.mkdir('repo') apkfile = 'repo/badzip_1.apk' with open(apkfile, 'w') as fp: fp.write('this is not a zip file') with self.assertRaises(fdroidserver.exception.BuildException): fdroidserver.update.scan_apk(apkfile) def test_process_apk(self): def _build_yaml_representer(dumper, data): '''Creates a YAML representation of a Build instance''' return dumper.represent_dict(data) config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) config['ndk_paths'] = dict() fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.rename_apks = False fdroidserver.update.options.delete_unknown = True fdroidserver.update.options.allow_disabled_algorithms = False for icon_dir in fdroidserver.update.get_all_icon_dirs('repo'): if not os.path.exists(icon_dir): os.makedirs(icon_dir) knownapks = fdroidserver.common.KnownApks() apkList = ['../urzip.apk', '../org.dyndns.fules.ck_20.apk'] for apkName in apkList: _, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks, False) # Don't care about the date added to the repo and relative apkName self.assertEqual(datetime, type(apk['added'])) del apk['added'] del apk['apkName'] # ensure that icons have been extracted properly if apkName == '../urzip.apk': self.assertEqual(apk['icon'], 'info.guardianproject.urzip.100.png') if apkName == '../org.dyndns.fules.ck_20.apk': self.assertEqual(apk['icon'], 'org.dyndns.fules.ck.20.png') for density in fdroidserver.update.screen_densities: icon_path = os.path.join(fdroidserver.update.get_icon_dir('repo', density), apk['icon']) self.assertTrue(os.path.isfile(icon_path)) self.assertTrue(os.path.getsize(icon_path) > 1) savepath = os.path.join('metadata', 'apk', apk['packageName'] + '.yaml') # Uncomment to save APK metadata # with open(savepath, 'w') as f: # yaml.add_representer(fdroidserver.metadata.Build, _build_yaml_representer) # yaml.dump(apk, f, default_flow_style=False) with open(savepath, 'r') as f: from_yaml = yaml.load(f, Loader=FullLoader) self.maxDiff = None self.assertEqual(apk, from_yaml) def test_process_apk_signed_by_disabled_algorithms(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config config['ndk_paths'] = dict() fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.verbose = True fdroidserver.update.options.rename_apks = False fdroidserver.update.options.delete_unknown = True fdroidserver.update.options.allow_disabled_algorithms = False knownapks = fdroidserver.common.KnownApks() tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) print('tmptestsdir', tmptestsdir) os.chdir(tmptestsdir) os.mkdir('repo') os.mkdir('archive') # setup the repo, create icons dirs, etc. fdroidserver.update.process_apks({}, 'repo', knownapks) fdroidserver.update.process_apks({}, 'archive', knownapks) disabledsigs = ['org.bitbucket.tickytacky.mirrormirror_2.apk', ] for apkName in disabledsigs: shutil.copy(os.path.join(self.basedir, apkName), os.path.join(tmptestsdir, 'repo')) skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks, allow_disabled_algorithms=True, archive_bad_sig=False) self.assertFalse(skip) self.assertIsNotNone(apk) self.assertTrue(cachechanged) self.assertFalse(os.path.exists(os.path.join('archive', apkName))) self.assertTrue(os.path.exists(os.path.join('repo', apkName))) if os.path.exists('/usr/bin/apksigner') or 'apksigner' in config: print('SKIPPING: apksigner installed and it allows MD5 signatures') return javac = config['jarsigner'].replace('jarsigner', 'javac') v = subprocess.check_output([javac, '-version'], stderr=subprocess.STDOUT)[6:-1].decode('utf-8') if LooseVersion(v) < LooseVersion('1.8.0_132'): print('SKIPPING: running tests with old Java (' + v + ')') return # this test only works on systems with fully updated Java/jarsigner # that has MD5 listed in jdk.jar.disabledAlgorithms in java.security # https://blogs.oracle.com/java-platform-group/oracle-jre-will-no-longer-trust-md5-signed-code-by-default skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks, allow_disabled_algorithms=False, archive_bad_sig=True) self.assertTrue(skip) self.assertIsNone(apk) self.assertFalse(cachechanged) self.assertTrue(os.path.exists(os.path.join('archive', apkName))) self.assertFalse(os.path.exists(os.path.join('repo', apkName))) skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'archive', knownapks, allow_disabled_algorithms=False, archive_bad_sig=False) self.assertFalse(skip) self.assertIsNotNone(apk) self.assertTrue(cachechanged) self.assertTrue(os.path.exists(os.path.join('archive', apkName))) self.assertFalse(os.path.exists(os.path.join('repo', apkName))) # ensure that icons have been moved to the archive as well for density in fdroidserver.update.screen_densities: icon_path = os.path.join(fdroidserver.update.get_icon_dir('archive', density), apk['icon']) self.assertTrue(os.path.isfile(icon_path)) self.assertTrue(os.path.getsize(icon_path) > 1) badsigs = ['urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release-unsigned.apk', ] for apkName in badsigs: shutil.copy(os.path.join(self.basedir, apkName), os.path.join(tmptestsdir, 'repo')) skip, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks, allow_disabled_algorithms=False, archive_bad_sig=False) self.assertTrue(skip) self.assertIsNone(apk) self.assertFalse(cachechanged) def test_process_invalid_apk(self): os.chdir(os.path.join(localmodule, 'tests')) if os.path.basename(os.getcwd()) != 'tests': raise Exception('This test must be run in the "tests/" subdir') config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = fdroidserver.common.options fdroidserver.update.options.delete_unknown = False knownapks = fdroidserver.common.KnownApks() apk = 'fake.ota.update_1234.zip' # this is not an APK, scanning should fail (skip, apk, cachechanged) = fdroidserver.update.process_apk({}, apk, 'repo', knownapks, False) self.assertTrue(skip) self.assertIsNone(apk) self.assertFalse(cachechanged) def test_translate_per_build_anti_features(self): os.chdir(os.path.join(localmodule, 'tests')) testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo') shutil.copytree(os.path.join(self.basedir, 'metadata'), 'metadata') config = dict() fdroidserver.common.fill_config_defaults(config) config['ndk_paths'] = dict() fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.delete_unknown = True fdroidserver.update.options.rename_apks = False fdroidserver.update.options.allow_disabled_algorithms = False apps = fdroidserver.metadata.read_metadata() knownapks = fdroidserver.common.KnownApks() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) fdroidserver.update.translate_per_build_anti_features(apps, apks) self.assertEqual(len(apks), 17) foundtest = False for apk in apks: if apk['packageName'] == 'com.politedroid' and apk['versionCode'] == 3: antiFeatures = apk.get('antiFeatures') self.assertTrue('KnownVuln' in antiFeatures) self.assertEqual(3, len(antiFeatures)) foundtest = True self.assertTrue(foundtest) def test_create_metadata_from_template(self): tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) print('tmptestsdir', tmptestsdir) os.chdir(tmptestsdir) os.mkdir('repo') os.mkdir('metadata') shutil.copy(os.path.join(localmodule, 'tests', 'urzip.apk'), 'repo') config = dict() fdroidserver.common.fill_config_defaults(config) config['ndk_paths'] = dict() fdroidserver.common.config = config fdroidserver.update.config = config fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.delete_unknown = False fdroidserver.update.options.rename_apks = False fdroidserver.update.options.allow_disabled_algorithms = False knownapks = fdroidserver.common.KnownApks() apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) self.assertEqual(1, len(apks)) apk = apks[0] testfile = 'metadata/info.guardianproject.urzip.yml' # create empty 0 byte .yml file, run read_metadata, it should work open(testfile, 'a').close() apps = fdroidserver.metadata.read_metadata() self.assertEqual(1, len(apps)) os.remove(testfile) # test using internal template apps = fdroidserver.metadata.read_metadata() self.assertEqual(0, len(apps)) fdroidserver.update.create_metadata_from_template(apk) self.assertTrue(os.path.exists(testfile)) apps = fdroidserver.metadata.read_metadata() self.assertEqual(1, len(apps)) for app in apps.values(): self.assertEqual('urzip', app['Name']) self.assertEqual(1, len(app['Categories'])) break # test using external template.yml os.remove(testfile) self.assertFalse(os.path.exists(testfile)) shutil.copy(os.path.join(localmodule, 'examples', 'template.yml'), tmptestsdir) fdroidserver.update.create_metadata_from_template(apk) self.assertTrue(os.path.exists(testfile)) apps = fdroidserver.metadata.read_metadata() self.assertEqual(1, len(apps)) for app in apps.values(): self.assertEqual('urzip', app['Name']) self.assertEqual(1, len(app['Categories'])) self.assertEqual('Internet', app['Categories'][0]) break with open(testfile) as fp: data = yaml.load(fp, Loader=SafeLoader) self.assertEqual('urzip', data['Name']) self.assertEqual('urzip', data['Summary']) def test_has_known_vulnerability(self): good = [ 'org.bitbucket.tickytacky.mirrormirror_1.apk', 'org.bitbucket.tickytacky.mirrormirror_2.apk', 'org.bitbucket.tickytacky.mirrormirror_3.apk', 'org.bitbucket.tickytacky.mirrormirror_4.apk', 'org.dyndns.fules.ck_20.apk', 'urzip.apk', 'urzip-badcert.apk', 'urzip-badsig.apk', 'urzip-release.apk', 'urzip-release-unsigned.apk', 'repo/com.politedroid_3.apk', 'repo/com.politedroid_4.apk', 'repo/com.politedroid_5.apk', 'repo/com.politedroid_6.apk', 'repo/obb.main.oldversion_1444412523.apk', 'repo/obb.mainpatch.current_1619_another-release-key.apk', 'repo/obb.mainpatch.current_1619.apk', 'repo/obb.main.twoversions_1101613.apk', 'repo/obb.main.twoversions_1101615.apk', 'repo/obb.main.twoversions_1101617.apk', 'repo/urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk', ] for f in good: self.assertFalse(fdroidserver.update.has_known_vulnerability(f)) with self.assertRaises(fdroidserver.exception.FDroidException): fdroidserver.update.has_known_vulnerability('janus.apk') def test_get_apk_icon_when_src_is_none(self): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config # pylint: disable=protected-access icons_src = fdroidserver.update._get_apk_icons_src('urzip-release.apk', None) assert icons_src == {} def test_strip_and_copy_image(self): tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) in_file = os.path.join(self.basedir, 'metadata', 'info.guardianproject.urzip', 'en-US', 'images', 'icon.png') out_file = os.path.join(tmptestsdir, 'icon.png') fdroidserver.update._strip_and_copy_image(in_file, out_file) self.assertTrue(os.path.exists(out_file)) in_file = os.path.join(self.basedir, 'corrupt-featureGraphic.png') out_file = os.path.join(tmptestsdir, 'corrupt-featureGraphic.png') fdroidserver.update._strip_and_copy_image(in_file, out_file) self.assertFalse(os.path.exists(out_file)) def test_create_metadata_from_template_empty_keys(self): apk = {'packageName': 'rocks.janicerand'} with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): os.mkdir('metadata') with open('template.yml', 'w') as f: f.write(textwrap.dedent('''\ Disabled: License: AuthorName: AuthorEmail: AuthorWebSite: WebSite: SourceCode: IssueTracker: Translation: Changelog: Donate: FlattrID: LiberapayID: Bitcoin: Litecoin: Name: AutoName: Summary: RequiresRoot: RepoType: Repo: Binaries: Builds: ArchivePolicy: AutoUpdateMode: UpdateCheckMode: UpdateCheckIgnore: VercodeOperation: UpdateCheckName: UpdateCheckData: CurrentVersion: CurrentVersionCode: NoSourceSince: ''')) fdroidserver.update.create_metadata_from_template(apk) with open(os.path.join('metadata', 'rocks.janicerand.yml')) as f: metadata_content = yaml.load(f, Loader=SafeLoader) self.maxDiff = None self.assertDictEqual(metadata_content, {'ArchivePolicy': '', 'AuthorEmail': '', 'AuthorName': '', 'AuthorWebSite': '', 'AutoName': 'rocks.janicerand', 'AutoUpdateMode': '', 'Binaries': '', 'Bitcoin': '', 'Builds': '', 'Changelog': '', 'CurrentVersion': '', 'CurrentVersionCode': '', 'Disabled': '', 'Donate': '', 'FlattrID': '', 'IssueTracker': '', 'LiberapayID': '', 'License': '', 'Litecoin': '', 'Name': 'rocks.janicerand', 'NoSourceSince': '', 'Repo': '', 'RepoType': '', 'RequiresRoot': '', 'SourceCode': '', 'Summary': 'rocks.janicerand', 'Translation': '', 'UpdateCheckData': '', 'UpdateCheckIgnore': '', 'UpdateCheckMode': '', 'UpdateCheckName': '', 'VercodeOperation': '', 'WebSite': ''}) def test_insert_funding_yml_donation_links(self): testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) os.mkdir('build') content = textwrap.dedent(""" community_bridge: '' custom: [LINK1, LINK2] github: USERNAME issuehunt: USERNAME ko_fi: USERNAME liberapay: USERNAME open_collective: USERNAME otechie: USERNAME patreon: USERNAME """) app = fdroidserver.metadata.App() app.id = 'fake.app.id' apps = {app.id: app} os.mkdir(os.path.join('build', app.id)) fdroidserver.update.insert_funding_yml_donation_links(apps) for field in DONATION_FIELDS: self.assertFalse(app.get(field)) with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp: fp.write(content) fdroidserver.update.insert_funding_yml_donation_links(apps) for field in DONATION_FIELDS: self.assertIsNotNone(app.get(field), field) self.assertEqual('LINK1', app.get('Donate')) self.assertEqual('USERNAME', app.get('Liberapay')) self.assertEqual('USERNAME', app.get('OpenCollective')) app['Donate'] = 'keepme' app['Liberapay'] = 'keepme' app['OpenCollective'] = 'keepme' fdroidserver.update.insert_funding_yml_donation_links(apps) for field in DONATION_FIELDS: self.assertEqual('keepme', app.get(field)) def test_insert_funding_yml_donation_links_one_at_a_time(self): """Exercise the FUNDING.yml code one entry at a time""" testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) os.mkdir('build') app = fdroidserver.metadata.App() app.id = 'fake.app.id' apps = {app.id: app} os.mkdir(os.path.join('build', app.id)) fdroidserver.update.insert_funding_yml_donation_links(apps) for field in DONATION_FIELDS: self.assertIsNone(app.get(field)) content = textwrap.dedent(""" community_bridge: 'blah-de-blah' github: USERNAME issuehunt: USERNAME ko_fi: USERNAME liberapay: USERNAME open_collective: USERNAME patreon: USERNAME """) for line in content.split('\n'): if not line: continue app = fdroidserver.metadata.App() app.id = 'fake.app.id' apps = {app.id: app} with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp: fp.write(line) data = yaml.load(line, Loader=SafeLoader) fdroidserver.update.insert_funding_yml_donation_links(apps) if 'liberapay' in data: self.assertEqual(data['liberapay'], app.get('Liberapay')) elif 'open_collective' in data: self.assertEqual(data['open_collective'], app.get('OpenCollective')) else: for v in data.values(): self.assertEqual(app.get('Donate', '').split('/')[-1], v) def test_insert_funding_yml_donation_links_with_corrupt_file(self): testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) os.mkdir('build') app = fdroidserver.metadata.App() app.id = 'fake.app.id' apps = {app.id: app} os.mkdir(os.path.join('build', app.id)) with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp: fp.write(textwrap.dedent(""" opencollective: foo custom: [] liberapay: : """)) fdroidserver.update.insert_funding_yml_donation_links(apps) for field in DONATION_FIELDS: self.assertIsNone(app.get(field)) def test_sanitize_funding_yml(self): with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp: data = yaml.load(fp, Loader=SafeLoader) for k, entries in data.items(): for entry in entries: if k in 'custom': m = fdroidserver.update.sanitize_funding_yml_entry(entry) else: m = fdroidserver.update.sanitize_funding_yml_name(entry) if k == 'bad': self.assertIsNone(m) else: self.assertIsNotNone(m) self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry('foo\nbar')) self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry( ''.join(chr(random.randint(65, 90)) for _ in range(2049)))) # not recommended but valid entries self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(12345)) self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(5.0)) self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(' WhyIncludeWhitespace ')) self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(['first', 'second'])) if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) parser = optparse.OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(UpdateTest)) unittest.main(failfast=False)