diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10d08696..5a40719f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,14 +5,19 @@ variables: GIT_DEPTH: 1 -test: +ci-images-base run-tests: image: registry.gitlab.com/fdroid/ci-images-base script: - $pip install -e .[test] - # the `fdroid build` test in tests/run-tests needs android-23 - - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platforms;android-23" > /dev/null - - cd tests - - ./complete-ci-tests + - ./tests/run-tests + # make sure that translations do not cause stacktraces + - cd $CI_PROJECT_DIR/locale + - for locale in *; do + test -d $locale || continue; + for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $CI_PROJECT_DIR/fdroid`; do + LANGUAGE=$locale $CI_PROJECT_DIR/fdroid $cmd --help > /dev/null; + done + done # Test that the parsing of the .yml metadata format didn't change from last # released version. This uses the commit ID of the release tags, diff --git a/MANIFEST.in b/MANIFEST.in index 1a9469a6..7f478ba8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -542,7 +542,6 @@ include tests/build-tools/generate.sh include tests/check-fdroid-apk include tests/checkupdates.TestCase include tests/common.TestCase -include tests/complete-ci-tests include tests/config.py include tests/corrupt-featureGraphic.png include tests/deploy.TestCase diff --git a/README.md b/README.md index e1d9b970..19b19a08 100644 --- a/README.md +++ b/README.md @@ -56,11 +56,10 @@ The test suite for all of the `fdroid` commands is in the _tests/_ subdir. _.gitlab-ci.yml_ and _.travis.yml_ run this test suite on various configurations. -* _tests/complete-ci-tests_ runs _pylint_ and all tests on two - different pyvenvs * _tests/run-tests_ runs the whole test suite * _tests/*.TestCase_ are individual unit tests for all of the `fdroid` commands, which can be run separately, e.g. `./update.TestCase`. +* run one test: `tests/common.TestCase CommonTest.test_get_apk_id` #### Additional tests for different linux distributions diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py index f1e83c90..84318804 100755 --- a/fdroidserver/btlog.py +++ b/fdroidserver/btlog.py @@ -49,9 +49,9 @@ from .exception import FDroidException options = None -def make_binary_transparency_log(repodirs, btrepo='binary_transparency', - url=None, - commit_title='fdroid update'): +def make_binary_transparency_log( + repodirs, btrepo='binary_transparency', url=None, commit_title='fdroid update' +): '''Log the indexes in a standalone git repo to serve as a "binary transparency" log. @@ -103,7 +103,7 @@ For more info on this idea: output = json.load(fp, object_pairs_hook=collections.OrderedDict) with open(dest, 'w') as fp: json.dump(output, fp, indent=2) - gitrepo.index.add([repof, ]) + gitrepo.index.add([repof]) for f in ('index.jar', 'index-v1.jar'): repof = os.path.join(repodir, f) if not os.path.exists(repof): @@ -116,7 +116,7 @@ For more info on this idea: jarout.writestr(info, jarin.read(info.filename)) jarout.close() jarin.close() - gitrepo.index.add([repof, ]) + gitrepo.index.add([repof]) output_files = [] for root, dirs, files in os.walk(repodir): @@ -137,10 +137,10 @@ For more info on this idea: fslogfile = os.path.join(cpdir, 'filesystemlog.json') with open(fslogfile, 'w') as fp: json.dump(output, fp, indent=2) - gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) + gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json')]) for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')): - gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ]) + gitrepo.index.add([os.path.join(repodir, os.path.basename(f))]) gitrepo.index.commit(commit_title) @@ -168,7 +168,8 @@ def main(): if not os.path.exists(options.git_repo): raise FDroidException( - '"%s" does not exist! Create it, or use --git-repo' % options.git_repo) + '"%s" does not exist! Create it, or use --git-repo' % options.git_repo + ) session = requests.Session() @@ -186,9 +187,7 @@ def main(): dlurl = options.url + '/' + repodir + '/' + f http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json') - headers = { - 'User-Agent': 'F-Droid 0.102.3' - } + headers = {'User-Agent': 'F-Droid 0.102.3'} etag = None if os.path.exists(http_headers_file): with open(http_headers_file) as fp: @@ -196,7 +195,9 @@ def main(): r = session.head(dlurl, headers=headers, allow_redirects=False) if r.status_code != 200: - logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl) + logging.debug( + 'HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl + ) continue if etag and etag == r.headers.get('ETag'): logging.debug('ETag matches, did not download ' + dlurl) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 9184b2c4..79c77456 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -262,6 +262,11 @@ def fill_config_defaults(thisconfig): if 'keytool' not in thisconfig and shutil.which('keytool'): thisconfig['keytool'] = shutil.which('keytool') + # enable apksigner by default so v2/v3 APK signatures validate + find_apksigner(thisconfig) + if not thisconfig.get('apksigner'): + logging.warning(_('apksigner not found! Cannot sign or verify modern APKs')) + for k in ['ndk_paths', 'java_paths']: d = thisconfig[k] for k2 in d.copy(): @@ -456,26 +461,35 @@ def assert_config_keystore(config): + "you can create one using: fdroid update --create-key") -def find_apksigner(): - """ +def find_apksigner(config): + """Searches for the best version apksigner and adds it to the config + Returns the best version of apksigner following this algorithm: * use config['apksigner'] if set * try to find apksigner in path * find apksigner in build-tools starting from newest installed going down to MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION :return: path to apksigner or None if no version is found + """ - if set_command_in_config('apksigner'): - return config['apksigner'] - build_tools_path = os.path.join(config['sdk_path'], 'build-tools') + command = 'apksigner' + if command in config: + return + + tmp = find_command(command) + if tmp is not None: + config[command] = tmp + return + + build_tools_path = os.path.join(config.get('sdk_path', ''), 'build-tools') if not os.path.isdir(build_tools_path): - return None + return for f in sorted(os.listdir(build_tools_path), reverse=True): if not os.path.isdir(os.path.join(build_tools_path, f)): continue try: if LooseVersion(f) < LooseVersion(MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION): - return None + return except TypeError: continue if os.path.exists(os.path.join(build_tools_path, f, 'apksigner')): @@ -483,7 +497,6 @@ def find_apksigner(): logging.info("Using %s " % apksigner) # memoize result config['apksigner'] = apksigner - return config['apksigner'] def find_sdk_tools_cmd(cmd): @@ -3001,7 +3014,7 @@ def _zipalign(unsigned_apk, aligned_apk): def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest): - """Implats a signature from metadata into an APK. + """Implants a signature from metadata into an APK. Note: this changes there supplied APK in place. So copy it if you need the original to be preserved. @@ -3061,82 +3074,48 @@ def get_min_sdk_version(apk): def sign_apk(unsigned_path, signed_path, keyalias): """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned - Use apksigner for making v2 and v3 signature for apks with targetSDK >=30 as - otherwise they won't be installable on Android 11/R. + NONE is a Java keyword used to configure smartcards as the + keystore. Otherwise, the keystore is a local file. + https://docs.oracle.com/javase/7/docs/technotes/guides/security/p11guide.html#KeyToolJarSigner - Otherwise use jarsigner for v1 only signatures until we have apksig v2/v3 - signature transplantig support. - - When using jarsigner we need to manually select the hash algorithm, - apksigner does this automatically. Apksigner also does the zipalign for us. - - SHA-256 support was added in android-18 (4.3), before then, the only options were MD5 - and SHA1. This aims to use SHA-256 when the APK does not target - older Android versions, and is therefore safe to do so. - - https://issuetracker.google.com/issues/36956587 - https://android-review.googlesource.com/c/platform/libcore/+/44491 + When using smartcards, apksigner does not use the same options has + Java/keytool/jarsigner (-providerName, -providerClass, + -providerArg, -storetype). apksigner documents the options as + --ks-provider-class and --ks-provider-arg. Those seem to be + accepted but fail when actually making a signature with weird + internal exceptions. We use the options that actually work. From: + https://geoffreymetais.github.io/code/key-signing/#scripting """ - apk = _get_androguard_APK(unsigned_path) - if apk.get_effective_target_sdk_version() >= 30: - if config['keystore'] == 'NONE': - # NOTE: apksigner doesn't like -providerName/--provider-name at all, don't use that. - # apksigner documents the options as --ks-provider-class and --ks-provider-arg - # those seem to be accepted but fail when actually making a signature with - # weird internal exceptions. Those options actually work. - # From: https://geoffreymetais.github.io/code/key-signing/#scripting - apksigner_smartcardoptions = config['smartcardoptions'].copy() - if '-providerName' in apksigner_smartcardoptions: - pos = config['smartcardoptions'].index('-providerName') - # remove -providerName and it's argument - del apksigner_smartcardoptions[pos] - del apksigner_smartcardoptions[pos] - replacements = {'-storetype': '--ks-type', - '-providerClass': '--provider-class', - '-providerArg': '--provider-arg'} - signing_args = [replacements.get(n, n) for n in apksigner_smartcardoptions] - else: - signing_args = ['--key-pass', 'env:FDROID_KEY_PASS'] - if not find_apksigner(): - raise BuildException(_("apksigner not found, it's required for signing!")) - cmd = [find_apksigner(), 'sign', - '--ks', config['keystore'], - '--ks-pass', 'env:FDROID_KEY_STORE_PASS'] - cmd += signing_args - cmd += ['--ks-key-alias', keyalias, - '--in', unsigned_path, - '--out', signed_path] - p = FDroidPopen(cmd, envs={ - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config.get('keypass', "")}) - if p.returncode != 0: - raise BuildException(_("Failed to sign application"), p.output) - os.remove(unsigned_path) + if config['keystore'] == 'NONE': + apksigner_smartcardoptions = config['smartcardoptions'].copy() + if '-providerName' in apksigner_smartcardoptions: + pos = config['smartcardoptions'].index('-providerName') + # remove -providerName and it's argument + del apksigner_smartcardoptions[pos] + del apksigner_smartcardoptions[pos] + replacements = {'-storetype': '--ks-type', + '-providerClass': '--provider-class', + '-providerArg': '--provider-arg'} + signing_args = [replacements.get(n, n) for n in apksigner_smartcardoptions] else: - - if get_min_sdk_version(apk) < 18: - signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1'] - else: - signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256'] - if config['keystore'] == 'NONE': - signing_args = config['smartcardoptions'] - else: - signing_args = ['-keypass:env', 'FDROID_KEY_PASS'] - - cmd = [config['jarsigner'], '-keystore', config['keystore'], - '-storepass:env', 'FDROID_KEY_STORE_PASS'] - cmd += signing_args - cmd += signature_algorithm - cmd += [unsigned_path, keyalias] - p = FDroidPopen(cmd, envs={ - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config.get('keypass', "")}) - if p.returncode != 0: - raise BuildException(_("Failed to sign application"), p.output) - - _zipalign(unsigned_path, signed_path) - os.remove(unsigned_path) + signing_args = ['--key-pass', 'env:FDROID_KEY_PASS'] + apksigner = config.get('apksigner', '') + if not shutil.which(apksigner): + raise BuildException(_("apksigner not found, it's required for signing!")) + cmd = [apksigner, 'sign', + '--ks', config['keystore'], + '--ks-pass', 'env:FDROID_KEY_STORE_PASS'] + cmd += signing_args + cmd += ['--ks-key-alias', keyalias, + '--in', unsigned_path, + '--out', signed_path] + p = FDroidPopen(cmd, envs={ + 'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config.get('keypass', "")}) + if p.returncode != 0: + raise BuildException(_("Failed to sign application"), p.output) + os.remove(unsigned_path) def verify_apks(signed_apk, unsigned_apk, tmp_dir): diff --git a/fdroidserver/exception.py b/fdroidserver/exception.py index f9f876ce..15228797 100644 --- a/fdroidserver/exception.py +++ b/fdroidserver/exception.py @@ -1,5 +1,4 @@ class FDroidException(Exception): - def __init__(self, value=None, detail=None): self.value = value self.detail = detail @@ -22,12 +21,14 @@ class FDroidException(Exception): else: ret = str(self.value) if self.detail: - ret += "\n==== detail begin ====\n%s\n==== detail end ====" % ''.join(self.detail).strip() + ret += ( + "\n==== detail begin ====\n%s\n==== detail end ====" + % ''.join(self.detail).strip() + ) return ret class MetaDataException(Exception): - def __init__(self, value): self.value = value diff --git a/fdroidserver/gpgsign.py b/fdroidserver/gpgsign.py index 86258b63..b6ee8fb7 100644 --- a/fdroidserver/gpgsign.py +++ b/fdroidserver/gpgsign.py @@ -73,9 +73,7 @@ def main(): sigpath = os.path.join(output_dir, sigfilename) if not os.path.exists(sigpath): - gpgargs = ['gpg', '-a', - '--output', sigpath, - '--detach-sig'] + gpgargs = ['gpg', '-a', '--output', sigpath, '--detach-sig'] if 'gpghome' in config: gpgargs.extend(['--homedir', config['gpghome']]) if 'gpgkey' in config: diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 64809771..8291612d 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -146,12 +146,6 @@ def main(): # now that we have a local config.yml, read configuration... config = common.read_config(options) - # enable apksigner by default so v2/v3 APK signatures validate - if common.find_apksigner() is not None: - apksigner = common.find_apksigner() - test_config['apksigner'] = apksigner - common.write_to_config(test_config, 'apksigner', apksigner) - # the NDK is optional and there may be multiple versions of it, so it's # left for the user to configure diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 41d70300..5e4ccbee 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -146,6 +146,9 @@ def status_update_json(generatedKeys, signedApks): logging.debug(_('Outputting JSON')) output = common.setup_status_output(start_timestamp) + output['apksigner'] = shutil.which(config.get('apksigner', '')) + output['jarsigner'] = shutil.which(config.get('jarsigner', '')) + output['keytool'] = shutil.which(config.get('keytool', '')) if generatedKeys: output['generatedKeys'] = generatedKeys if signedApks: diff --git a/fdroidserver/signindex.py b/fdroidserver/signindex.py index 2c5859d5..a993ae73 100644 --- a/fdroidserver/signindex.py +++ b/fdroidserver/signindex.py @@ -99,7 +99,8 @@ def main(): if 'jarsigner' not in config: raise FDroidException( - _('Java jarsigner not found! Install in standard location or set java_paths!')) + _('Java jarsigner not found! Install in standard location or set java_paths!') + ) repodirs = ['repo'] if config['archive_older'] != 0: diff --git a/fdroidserver/tail.py b/fdroidserver/tail.py index fb441f97..1f64e5df 100644 --- a/fdroidserver/tail.py +++ b/fdroidserver/tail.py @@ -33,6 +33,7 @@ import threading class Tail(object): ''' Represents a tail command. ''' + def __init__(self, tailed_file): ''' Initiate a Tail instance. Check for file validity, assigns callback function to standard out. @@ -95,7 +96,6 @@ class Tail(object): class TailError(Exception): - def __init__(self, msg): self.message = msg diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 988c65a2..1cd94e47 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -144,6 +144,9 @@ def status_update_json(apps, apks): output['noPackages'] = [] output['needsUpdate'] = [] output['noUpdateCheck'] = [] + output['apksigner'] = shutil.which(config.get('apksigner', '')) + output['jarsigner'] = shutil.which(config.get('jarsigner', '')) + output['keytool'] = shutil.which(config.get('keytool', '')) for appid in apps: app = apps[appid] @@ -1395,17 +1398,17 @@ def scan_apk(apk_file): logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file))) apk['sig'] = getsig(apk_file) if not apk['sig']: - raise BuildException("Failed to get apk signature") + raise BuildException(_("Failed to get APK signing key fingerprint")) apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(), apk_file)) if not apk.get('signer'): - raise BuildException("Failed to get apk signing key fingerprint") + raise BuildException(_("Failed to get APK signing key fingerprint")) # Get size of the APK apk['size'] = os.path.getsize(apk_file) if 'minSdkVersion' not in apk: - logging.warning("No SDK version information found in {0}".format(apk_file)) + logging.warning(_("No minimum SDK version found in {0}, using default (3).").format(apk_file)) apk['minSdkVersion'] = 3 # aapt defaults to 3 as the min # Check for known vulnerabilities @@ -1524,13 +1527,16 @@ def scan_apk_androguard(apk, apkfile): icon_id_str = apkobject.get_element("application", "icon") if icon_id_str: icon_id = int(icon_id_str.replace("@", "0x"), 16) - resource_id = arsc.get_id(apk['packageName'], icon_id) - if resource_id: - icon_name = arsc.get_id(apk['packageName'], icon_id)[1] - else: - # don't use 'anydpi' aka 0xFFFE aka 65534 since it is XML - icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon(max_dpi=65534 - 1)))[0] - apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name) + try: + resource_id = arsc.get_id(apk['packageName'], icon_id) + if resource_id: + icon_name = arsc.get_id(apk['packageName'], icon_id)[1] + else: + # don't use 'anydpi' aka 0xFFFE aka 65534 since it is XML + icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon(max_dpi=65534 - 1)))[0] + apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name) + except Exception as e: + logging.error("Cannot fetch icon from %s: %s" % (apkfile, str(e))) arch_re = re.compile("^lib/(.*)/.*$") arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)]) diff --git a/tests/build.TestCase b/tests/build.TestCase index a696f196..da8bd7d2 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -37,6 +37,8 @@ class BuildTest(unittest.TestCase): if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) os.chdir(self.basedir) + fdroidserver.common.config = None + fdroidserver.build.config = None def test_get_apk_metadata(self): config = dict() diff --git a/tests/common.TestCase b/tests/common.TestCase index abb8ee51..607de389 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -619,12 +619,9 @@ class CommonTest(unittest.TestCase): def test_sign_apk(self): fdroidserver.common.config = None config = fdroidserver.common.read_config(fdroidserver.common.options) - try: - fdroidserver.common.find_sdk_tools_cmd('zipalign') - except fdroidserver.exception.FDroidException: - self.skipTest('SKIPPING test_sign_apk, zipalign not installed!') + if 'apksigner' not in config: + self.skipTest('SKIPPING test_sign_apk, apksigner not installed!') - config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') config['keyalias'] = 'sova' config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' @@ -656,18 +653,6 @@ class CommonTest(unittest.TestCase): self.assertTrue(fdroidserver.common.verify_apk_signature(signed)) self.assertEqual('18', fdroidserver.common._get_androguard_APK(signed).get_min_sdk_version()) - def test_sign_apk_targetsdk_30(self): - fdroidserver.common.config = None - config = fdroidserver.common.read_config(fdroidserver.common.options) - if not fdroidserver.common.find_apksigner(): - self.skipTest('SKIPPING as apksigner is not installed!') - config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') - config['keyalias'] = 'sova' - config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' - config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' - config['keystore'] = os.path.join(self.basedir, 'keystore.jks') - testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) - shutil.copy(os.path.join(self.basedir, 'minimal_targetsdk_30_unsigned.apk'), testdir) unsigned = os.path.join(testdir, 'minimal_targetsdk_30_unsigned.apk') signed = os.path.join(testdir, 'minimal_targetsdk_30.apk') @@ -681,18 +666,6 @@ class CommonTest(unittest.TestCase): # verify it has a v2 signature self.assertTrue(fdroidserver.common._get_androguard_APK(signed).is_signed_v2()) - def test_sign_no_targetsdk(self): - fdroidserver.common.config = None - config = fdroidserver.common.read_config(fdroidserver.common.options) - if not fdroidserver.common.find_apksigner(): - self.skipTest('SKIPPING as apksigner is not installed!') - config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') - config['keyalias'] = 'sova' - config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' - config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' - config['keystore'] = os.path.join(self.basedir, 'keystore.jks') - testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) - shutil.copy(os.path.join(self.basedir, 'no_targetsdk_minsdk30_unsigned.apk'), testdir) unsigned = os.path.join(testdir, 'no_targetsdk_minsdk30_unsigned.apk') signed = os.path.join(testdir, 'no_targetsdk_minsdk30_signed.apk') @@ -1442,7 +1415,7 @@ class CommonTest(unittest.TestCase): self.assertFalse(os.path.exists('config.yml')) self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config(fdroidserver.common.options) - self.assertEqual(None, config.get('apksigner')) + self.assertIsNone(config.get('stats_server')) self.assertIsNotNone(config.get('char_limits')) def test_with_config_yml(self): diff --git a/tests/complete-ci-tests b/tests/complete-ci-tests deleted file mode 100755 index 07e0b2a0..00000000 --- a/tests/complete-ci-tests +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -# -# this is the script run by the Jenkins and gitlab-ci continuous integration -# build services. It is a thorough set of tests that runs all the tests using -# the various methods of installing/running fdroidserver. It is separate from -# ./tests/run-tests because its too heavy for manual use. - -if [ `dirname $0` != "." ]; then - echo "only run this script like ./`basename $0`" - exit 1 -fi - -set -e -set -x - -if [ -z $WORKSPACE ]; then - WORKSPACE=`cd $(dirname $0)/.. && pwd` - echo "Setting Workspace to $WORKSPACE" -fi - -if [ -z $ANDROID_HOME ]; then - if [ -e ~/.android/bashrc ]; then - . ~/.android/bashrc - else - echo "ANDROID_HOME must be set!" - exit 1 - fi -fi - -if ! which pyvenv; then - echo "pyvenv required to run this test suite!" - exit 1 -fi - -apksource=$1 - -#------------------------------------------------------------------------------# -# cache pypi downloads -if [ -z $PIP_DOWNLOAD_CACHE ]; then - export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache -fi - -#------------------------------------------------------------------------------# -# run local tests, don't scan fdroidserver/ project for APKs - -cd $WORKSPACE/tests -./run-tests $apksource - - -#------------------------------------------------------------------------------# -# make sure that translations do not cause stacktraces -cd $WORKSPACE/locale -for locale in *; do - if [ ! -d $locale ]; then - continue - fi - for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $WORKSPACE/fdroid`; do - LANGUAGE=$locale $WORKSPACE/fdroid $cmd --help > /dev/null - done -done - - -#------------------------------------------------------------------------------# -# test install using install direct from git repo -cd $WORKSPACE -rm -rf $WORKSPACE/env -pyvenv $WORKSPACE/env -. $WORKSPACE/env/bin/activate -pip3 install --quiet -e $WORKSPACE[test] -python3 setup.py compile_catalog install - -# make sure translation files were installed -test -e $WORKSPACE/env/share/locale/de/LC_MESSAGES/fdroidserver.mo - -# run tests in new pip+pyvenv install -fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource diff --git a/tests/init.TestCase b/tests/init.TestCase index 918643cc..1062214f 100755 --- a/tests/init.TestCase +++ b/tests/init.TestCase @@ -10,7 +10,6 @@ import shutil import sys import tempfile import unittest -import yaml localmodule = os.path.realpath( @@ -68,11 +67,7 @@ class InitTest(unittest.TestCase): sys.argv = ['fdroid init', '--keystore', 'keystore.jks', '--repo-keyalias=sova'] fdroidserver.init.main() - with open('config.yml') as fp: - config = yaml.safe_load(fp) - self.assertTrue(os.path.exists(config['keystore'])) - self.assertTrue(os.path.exists(config['apksigner'])) - self.assertEqual(apksigner, config['apksigner']) + self.assertEqual(apksigner, fdroidserver.init.config.get('apksigner')) if __name__ == "__main__": diff --git a/tests/publish.TestCase b/tests/publish.TestCase index d0d08376..f57dc955 100755 --- a/tests/publish.TestCase +++ b/tests/publish.TestCase @@ -11,6 +11,7 @@ # import inspect +import json import logging import optparse import os @@ -221,6 +222,34 @@ class PublishTest(unittest.TestCase): self.assertTrue(pk.is_decrypted()) self.assertEqual(jks.util.RSA_ENCRYPTION_OID, pk.algorithm_oid) + def test_status_update_json(self): + common.config = {} + publish.config = {} + with tempfile.TemporaryDirectory() as tmpdir: + os.chdir(tmpdir) + with mock.patch('sys.argv', ['fdroid publish', '']): + publish.status_update_json([], []) + with open('repo/status/publish.json') as fp: + data = json.load(fp) + self.assertTrue('apksigner' in data) + + publish.config = { + 'apksigner': 'apksigner', + } + publish.status_update_json([], []) + with open('repo/status/publish.json') as fp: + data = json.load(fp) + self.assertEqual(shutil.which(publish.config['apksigner']), data['apksigner']) + + publish.config = {} + common.fill_config_defaults(publish.config) + publish.status_update_json([], []) + with open('repo/status/publish.json') as fp: + data = json.load(fp) + self.assertEqual(publish.config.get('apksigner'), data['apksigner']) + self.assertEqual(publish.config['jarsigner'], data['jarsigner']) + self.assertEqual(publish.config['keytool'], data['keytool']) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/run-tests b/tests/run-tests index 6058cd85..ccf1bd40 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -72,11 +72,17 @@ have_git_2_3() { is_MD5_disabled() { javac $WORKSPACE/tests/IsMD5Disabled.java && java -cp $WORKSPACE/tests IsMD5Disabled - return $? } use_apksigner() { - test -x "`sed -En 's,^ *apksigner: +,,p' config.yml`" + python3 -c " +import sys +sys.path.insert(0, '$WORKSPACE') +from fdroidserver import common +c = {'sdk_path': '$ANDROID_HOME'} +common.find_apksigner(c) +exit(c.get('apksigner') is None) +" } #------------------------------------------------------------------------------# @@ -169,7 +175,7 @@ $fdroid --version #------------------------------------------------------------------------------# echo_header 'run process when building and signing are on separate machines' -if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then +if use_apksigner; then REPOROOT=`create_test_dir` cd $REPOROOT cp $WORKSPACE/tests/keystore.jks $REPOROOT/ @@ -213,20 +219,6 @@ $fdroid readmeta $fdroid update -#------------------------------------------------------------------------------# -echo_header 'run "fdroid build" in fresh git checkout from import.TestCase' - -cd $WORKSPACE/tests/tmp/importer -git remote update -p -git clean -fdx -# stick with known working commit, in case future commits break things for this code -git reset --hard fea54e1161d5eb9eb1a54e26253ef84d3ab63705 -if [ -d $ANDROID_HOME/platforms/android-23 && -d $ANDROID_HOME/build-tools/23.0.3 ]; then - $fdroid build --verbose org.fdroid.ci.test.app:300 -else - echo 'WARNING: Skipping "fdroid build" test since android-23 is missing!' -fi - #------------------------------------------------------------------------------# echo_header 'copy git import and run "fdroid scanner" on it' @@ -325,8 +317,9 @@ else test `grep '' repo/index.xml | wc -l` -eq 7 fi + #------------------------------------------------------------------------------# -if ! which apksigner; then +if ! use_apksigner; then echo_header 'test per-app "Archive Policy"' REPOROOT=`create_test_dir` @@ -534,7 +527,7 @@ test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk test -e archive/urzip-badsig.apk sed -i.tmp '/apksigner:/d' config.yml -if ! which apksigner; then +if ! use_apksigner; then $sed -i.tmp '/allow_disabled_algorithms/d' config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 5 @@ -1240,28 +1233,30 @@ fi #------------------------------------------------------------------------------# echo_header 'test extracting and publishing with developer signature' -REPOROOT=`create_test_dir` -cd $REPOROOT -fdroid_init_with_prebuilt_keystore -echo 'keydname: "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.yml -test -d metadata || mkdir metadata -cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ -test -d repo || mkdir repo -test -d unsigned || mkdir unsigned -cp $WORKSPACE/tests/repo/com.politedroid_6.apk unsigned/ -$fdroid signatures unsigned/com.politedroid_6.apk -test -d metadata/com.politedroid/signatures/6 -test -f metadata/com.politedroid/signatures/6/MANIFEST.MF -test -f metadata/com.politedroid/signatures/6/RELEASE.RSA -test -f metadata/com.politedroid/signatures/6/RELEASE.SF -! test -f repo/com.politedroid_6.apk -$fdroid publish -test -f repo/com.politedroid_6.apk -if which jarsigner; then - jarsigner -verify repo/com.politedroid_6.apk -fi -if which apksigner; then - apksigner verify repo/com.politedroid_6.apk +if use_apksigner; then + REPOROOT=`create_test_dir` + cd $REPOROOT + fdroid_init_with_prebuilt_keystore + echo 'keydname: "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.yml + test -d metadata || mkdir metadata + cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ + test -d repo || mkdir repo + test -d unsigned || mkdir unsigned + cp $WORKSPACE/tests/repo/com.politedroid_6.apk unsigned/ + $fdroid signatures unsigned/com.politedroid_6.apk + test -d metadata/com.politedroid/signatures/6 + test -f metadata/com.politedroid/signatures/6/MANIFEST.MF + test -f metadata/com.politedroid/signatures/6/RELEASE.RSA + test -f metadata/com.politedroid/signatures/6/RELEASE.SF + ! test -f repo/com.politedroid_6.apk + $fdroid publish + test -f repo/com.politedroid_6.apk + if which apksigner; then + apksigner verify repo/com.politedroid_6.apk + fi + if which jarsigner; then + jarsigner -verify repo/com.politedroid_6.apk + fi fi diff --git a/tests/update.TestCase b/tests/update.TestCase index 06d5892d..fbdccc5e 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -6,6 +6,7 @@ import copy import git import glob import inspect +import json import logging import optparse import os @@ -23,6 +24,7 @@ from binascii import unhexlify from datetime import datetime from distutils.version import LooseVersion from testcommon import TmpCwd +from unittest import mock try: from yaml import CSafeLoader as SafeLoader @@ -63,6 +65,7 @@ DONATION_FIELDS = ( class Options: allow_disabled_algorithms = False clean = False + pretty = True rename_apks = False @@ -656,9 +659,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) - apksigner = fdroidserver.common.find_apksigner() - if apksigner: - config['apksigner'] = apksigner + if 'apksigner' in config: apk_info = fdroidserver.update.scan_apk('v2.only.sig_2.apk') self.assertIsNone(apk_info.get('maxSdkVersion')) self.assertEqual(apk_info.get('versionName'), 'v2-only') @@ -1376,6 +1377,35 @@ class UpdateTest(unittest.TestCase): fdroidserver.update._set_author_entry(app, key, f) self.assertIsNone(app.get(key)) + def test_status_update_json(self): + fdroidserver.common.config = {} + fdroidserver.update.config = {} + fdroidserver.update.options = Options + with tempfile.TemporaryDirectory() as tmpdir: + os.chdir(tmpdir) + with mock.patch('sys.argv', ['fdroid update', '']): + fdroidserver.update.status_update_json([], []) + with open('repo/status/update.json') as fp: + data = json.load(fp) + self.assertTrue('apksigner' in data) + + fdroidserver.update.config = { + 'apksigner': 'apksigner', + } + fdroidserver.update.status_update_json([], []) + with open('repo/status/update.json') as fp: + data = json.load(fp) + self.assertEqual(shutil.which(fdroidserver.update.config['apksigner']), data['apksigner']) + + fdroidserver.update.config = {} + fdroidserver.common.fill_config_defaults(fdroidserver.update.config) + fdroidserver.update.status_update_json([], []) + with open('repo/status/update.json') as fp: + data = json.load(fp) + self.assertEqual(fdroidserver.update.config.get('apksigner'), data['apksigner']) + self.assertEqual(fdroidserver.update.config['jarsigner'], data['jarsigner']) + self.assertEqual(fdroidserver.update.config['keytool'], data['keytool']) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))