diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0de8ffaf..14807970 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,12 +66,17 @@ debian_testing: - apt-get install aapt androguard + apksigner fdroidserver git gnupg python3-defusedxml python3-setuptools zipalign + # Debian has apksigner depend on binfmt support which isn't very docker friendly + # We create a shell wrapper instead + - echo -e '#!/bin/sh\njava -jar /usr/lib/android-sdk/build-tools/debian/apksigner.jar "$@"' > /usr/local/bin/apksigner + - chmod +x /usr/local/bin/apksigner - python3 -c 'import fdroidserver' - python3 -c 'import androguard' - cd tests @@ -197,7 +202,7 @@ fedora_latest: - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip - unzip -q tools.zip - rm tools.zip - - export AAPT_VERSION=`sed -n "s,^MINIMUM_AAPT_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py` + - export BUILD_TOOLS_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py` - export JAVA_HOME=/etc/alternatives/jre - export ANDROID_HOME=`pwd`/android-sdk - mkdir $ANDROID_HOME @@ -209,7 +214,7 @@ fedora_latest: - mkdir ~/.android - touch ~/.android/repositories.cfg - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools" - - echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;$AAPT_VERSION" + - echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION" - chown -R testuser . - cd tests - su testuser --login --command diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e6b10305..586867a6 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -429,7 +429,11 @@ def find_apksigner(): if set_command_in_config('apksigner'): return config['apksigner'] build_tools_path = os.path.join(config['sdk_path'], 'build-tools') + if not os.path.isdir(build_tools_path): + return None for f in sorted(os.listdir(build_tools_path), reverse=True): + if not os.path.isdir(os.path.join(build_tools_path, f)): + continue if LooseVersion(f) < LooseVersion(MINIMUM_AAPT_BUILD_TOOLS_VERSION): return None if os.path.exists(os.path.join(build_tools_path, f, 'apksigner')): diff --git a/tests/common.TestCase b/tests/common.TestCase index eb67836c..98815d5b 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -675,6 +675,8 @@ class CommonTest(unittest.TestCase): def test_sign_apk_targetsdk_30(self): fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) + if not fdroidserver.common.find_apksigner(): + self.skipTest('SKIPPING as apksigner is not installed!') config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') config['keyalias'] = 'sova' config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 8ab502c4..c48bdba2 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -18,6 +18,11 @@ import textwrap from collections import OrderedDict from unittest import mock +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) print('localmodule: ' + localmodule) @@ -103,7 +108,7 @@ class MetadataTest(unittest.TestCase): def test_valid_funding_yml_regex(self): """Check the regex can find all the cases""" with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp: - data = yaml.safe_load(fp) + data = yaml.load(fp, Loader=SafeLoader) for k, entries in data.items(): for entry in entries: @@ -135,8 +140,8 @@ class MetadataTest(unittest.TestCase): frommeta = dict(apps[appid]) self.assertTrue(appid in apps) with open(savepath, 'r') as f: - frompickle = yaml.load(f) - self.assertEqual(frommeta, frompickle) + from_yaml = yaml.load(f, Loader=SafeLoader) + self.assertEqual(frommeta, from_yaml) # comment above assert and uncomment below to update test # files when new metadata fields are added # with open(savepath, 'w') as f: diff --git a/tests/update.TestCase b/tests/update.TestCase index eee01734..4f9c3f3f 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -21,6 +21,22 @@ from binascii import unhexlify from distutils.version import LooseVersion from testcommon import TmpCwd +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +try: + from yaml import CFullLoader as FullLoader +except ImportError: + try: + # FullLoader is available from PyYaml 5.1+, as we don't load user + # controlled data here, it's okay to fall back the unsafe older + # Loader + from yaml import FullLoader + except ImportError: + from yaml import Loader as FullLoader + localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) print('localmodule: ' + localmodule) @@ -207,6 +223,10 @@ class UpdateTest(unittest.TestCase): shutil.copytree(os.path.join(self.basedir, 'triple-t-2'), tmptestsdir) os.chdir(tmptestsdir) + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.update.config = config fdroidserver.update.options = fdroidserver.common.options apps = fdroidserver.metadata.read_metadata(xref=True) @@ -339,9 +359,10 @@ class UpdateTest(unittest.TestCase): def testScanApksAndObbs(self): os.chdir(os.path.join(localmodule, 'tests')) - if os.path.basename(os.getcwd()) != 'tests': - raise Exception('This test must be run in the "tests/" subdir') - + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo') + shutil.copytree(os.path.join(self.basedir, 'metadata'), 'metadata') config = dict() fdroidserver.common.fill_config_defaults(config) config['ndk_paths'] = dict() @@ -396,9 +417,9 @@ class UpdateTest(unittest.TestCase): def test_apkcache_json(self): """test the migration from pickle to json""" os.chdir(os.path.join(localmodule, 'tests')) - if os.path.basename(os.getcwd()) != 'tests': - raise Exception('This test must be run in the "tests/" subdir') - + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo') config = dict() fdroidserver.common.fill_config_defaults(config) config['ndk_paths'] = dict() @@ -437,9 +458,6 @@ class UpdateTest(unittest.TestCase): fdroidserver.common.config = config fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) - if os.path.basename(os.getcwd()) != 'tests': - raise Exception('This test must be run in the "tests/" subdir') - try: config['aapt'] = fdroidserver.common.find_sdk_tools_cmd('aapt') except fdroidserver.exception.FDroidException: @@ -466,17 +484,16 @@ class UpdateTest(unittest.TestCase): print('USE_ANDROGUARD', use_androguard) - try: - apksigner = fdroidserver.common.find_sdk_tools_cmd('apksigner') - if use_androguard and apksigner: # v2 parsing needs both + apksigner = fdroidserver.common.find_apksigner() + if apksigner: + if use_androguard: # v2 parsing needs both config['apksigner'] = apksigner apk_info = fdroidserver.update.scan_apk('v2.only.sig_2.apk') self.assertIsNone(apk_info.get('maxSdkVersion')) self.assertEqual(apk_info.get('versionName'), 'v2-only') self.assertEqual(apk_info.get('versionCode'), 2) - except fdroidserver.exception.FDroidException: + else: print('WARNING: skipping v2-only test since apksigner cannot be found') - apk_info = fdroidserver.update.scan_apk('repo/v1.v2.sig_1020.apk') self.assertIsNone(apk_info.get('maxSdkVersion')) self.assertEqual(apk_info.get('versionName'), 'v1+2') @@ -587,13 +604,10 @@ class UpdateTest(unittest.TestCase): fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) - if os.path.basename(os.getcwd()) != 'tests': - raise Exception('This test must be run in the "tests/" subdir') config['ndk_paths'] = dict() fdroidserver.common.config = config fdroidserver.update.config = config - fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.rename_apks = False @@ -634,7 +648,7 @@ class UpdateTest(unittest.TestCase): # yaml.dump(apk, f, default_flow_style=False) with open(savepath, 'r') as f: - from_yaml = yaml.load(f) + from_yaml = yaml.load(f, Loader=FullLoader) self.maxDiff = None self.assertEqual(apk, from_yaml) @@ -743,6 +757,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config fdroidserver.update.config = config + fdroidserver.update.options = fdroidserver.common.options fdroidserver.update.options.delete_unknown = False knownapks = fdroidserver.common.KnownApks() @@ -756,9 +771,10 @@ class UpdateTest(unittest.TestCase): def test_translate_per_build_anti_features(self): os.chdir(os.path.join(localmodule, 'tests')) - if os.path.basename(os.getcwd()) != 'tests': - raise Exception('This test must be run in the "tests/" subdir') - + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + shutil.copytree(os.path.join(self.basedir, 'repo'), 'repo') + shutil.copytree(os.path.join(self.basedir, 'metadata'), 'metadata') config = dict() fdroidserver.common.fill_config_defaults(config) config['ndk_paths'] = dict() @@ -844,7 +860,7 @@ class UpdateTest(unittest.TestCase): self.assertEqual('Internet', app['Categories'][0]) break with open(testfile) as fp: - data = yaml.load(fp) + data = yaml.load(fp, Loader=SafeLoader) self.assertEqual('urzip', data['Name']) self.assertEqual('urzip', data['Summary']) @@ -943,7 +959,7 @@ class UpdateTest(unittest.TestCase): ''')) fdroidserver.update.create_metadata_from_template(apk) with open(os.path.join('metadata', 'rocks.janicerand.yml')) as f: - metadata_content = yaml.load(f) + metadata_content = yaml.load(f, Loader=SafeLoader) self.maxDiff = None self.assertDictEqual(metadata_content, {'ArchivePolicy': '', @@ -1050,7 +1066,7 @@ class UpdateTest(unittest.TestCase): apps = {app.id: app} with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp: fp.write(line) - data = yaml.load(line) + data = yaml.load(line, Loader=SafeLoader) fdroidserver.update.insert_funding_yml_donation_links(apps) if 'liberapay' in data: self.assertEqual(data['liberapay'], app.get('Liberapay')) @@ -1080,7 +1096,7 @@ class UpdateTest(unittest.TestCase): def test_sanitize_funding_yml(self): with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp: - data = yaml.safe_load(fp) + data = yaml.load(fp, Loader=SafeLoader) for k, entries in data.items(): for entry in entries: if k in 'custom':