Merge branch 'apksigner-first' into 'master'

switch to using apksigner by default, as much as possible

Closes #880 and admin#202

See merge request fdroid/fdroidserver!889

.gitlab-ci.yml

@@ -5,14 +5,19 @@ variables:
   GIT_DEPTH: 1
 
 
-test:
+ci-images-base run-tests:
   image: registry.gitlab.com/fdroid/ci-images-base
   script:
     - $pip install -e .[test]
-    # the `fdroid build` test in tests/run-tests needs android-23
+    - ./tests/run-tests
-    - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platforms;android-23" > /dev/null
+    # make sure that translations do not cause stacktraces
-    - cd tests
+    - cd $CI_PROJECT_DIR/locale
-    - ./complete-ci-tests
+    - for locale in *; do
+          test -d $locale || continue;
+          for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $CI_PROJECT_DIR/fdroid`; do
+              LANGUAGE=$locale $CI_PROJECT_DIR/fdroid $cmd --help > /dev/null;
+          done
+      done
 
 # Test that the parsing of the .yml metadata format didn't change from last
 # released version. This uses the commit ID of the release tags,

MANIFEST.in

@@ -542,7 +542,6 @@ include tests/build-tools/generate.sh
 include tests/check-fdroid-apk
 include tests/checkupdates.TestCase
 include tests/common.TestCase
-include tests/complete-ci-tests
 include tests/config.py
 include tests/corrupt-featureGraphic.png
 include tests/deploy.TestCase

README.md

@@ -56,11 +56,10 @@ The test suite for all of the `fdroid` commands is in the _tests/_
 subdir.  _.gitlab-ci.yml_ and _.travis.yml_ run this test suite on
 various configurations.
 
-* _tests/complete-ci-tests_ runs _pylint_ and all tests on two
-  different pyvenvs
 * _tests/run-tests_ runs the whole test suite
 * _tests/*.TestCase_ are individual unit tests for all of the `fdroid`
   commands, which can be run separately, e.g. `./update.TestCase`.
+* run one test: `tests/common.TestCase CommonTest.test_get_apk_id`
 
 
 #### Additional tests for different linux distributions

fdroidserver/btlog.py

@@ -49,9 +49,9 @@ from .exception import FDroidException
 options = None
 
 
-def make_binary_transparency_log(repodirs, btrepo='binary_transparency',
+def make_binary_transparency_log(
-                                 url=None,
+    repodirs, btrepo='binary_transparency', url=None, commit_title='fdroid update'
-                                 commit_title='fdroid update'):
+):
     '''Log the indexes in a standalone git repo to serve as a "binary
     transparency" log.
 
@@ -103,7 +103,7 @@ For more info on this idea:
                     output = json.load(fp, object_pairs_hook=collections.OrderedDict)
                 with open(dest, 'w') as fp:
                     json.dump(output, fp, indent=2)
-            gitrepo.index.add([repof, ])
+            gitrepo.index.add([repof])
         for f in ('index.jar', 'index-v1.jar'):
             repof = os.path.join(repodir, f)
             if not os.path.exists(repof):
@@ -116,7 +116,7 @@ For more info on this idea:
                     jarout.writestr(info, jarin.read(info.filename))
             jarout.close()
             jarin.close()
-            gitrepo.index.add([repof, ])
+            gitrepo.index.add([repof])
 
         output_files = []
         for root, dirs, files in os.walk(repodir):
@@ -137,10 +137,10 @@ For more info on this idea:
         fslogfile = os.path.join(cpdir, 'filesystemlog.json')
         with open(fslogfile, 'w') as fp:
             json.dump(output, fp, indent=2)
-        gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
+        gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json')])
 
         for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
-            gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ])
+            gitrepo.index.add([os.path.join(repodir, os.path.basename(f))])
 
     gitrepo.index.commit(commit_title)
 
@@ -168,7 +168,8 @@ def main():
 
     if not os.path.exists(options.git_repo):
         raise FDroidException(
-            '"%s" does not exist! Create it, or use --git-repo' % options.git_repo)
+            '"%s" does not exist! Create it, or use --git-repo' % options.git_repo
+        )
 
     session = requests.Session()
 
@@ -186,9 +187,7 @@ def main():
             dlurl = options.url + '/' + repodir + '/' + f
             http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json')
 
-            headers = {
+            headers = {'User-Agent': 'F-Droid 0.102.3'}
-                'User-Agent': 'F-Droid 0.102.3'
-            }
             etag = None
             if os.path.exists(http_headers_file):
                 with open(http_headers_file) as fp:
@@ -196,7 +195,9 @@ def main():
 
             r = session.head(dlurl, headers=headers, allow_redirects=False)
             if r.status_code != 200:
-                logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl)
+                logging.debug(
+                    'HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl
+                )
                 continue
             if etag and etag == r.headers.get('ETag'):
                 logging.debug('ETag matches, did not download ' + dlurl)

fdroidserver/common.py

@@ -262,6 +262,11 @@ def fill_config_defaults(thisconfig):
     if 'keytool' not in thisconfig and shutil.which('keytool'):
         thisconfig['keytool'] = shutil.which('keytool')
 
+    # enable apksigner by default so v2/v3 APK signatures validate
+    find_apksigner(thisconfig)
+    if not thisconfig.get('apksigner'):
+        logging.warning(_('apksigner not found! Cannot sign or verify modern APKs'))
+
     for k in ['ndk_paths', 'java_paths']:
         d = thisconfig[k]
         for k2 in d.copy():
@@ -456,26 +461,35 @@ def assert_config_keystore(config):
                               + "you can create one using: fdroid update --create-key")
 
 
-def find_apksigner():
+def find_apksigner(config):
-    """
+    """Searches for the best version apksigner and adds it to the config
+
     Returns the best version of apksigner following this algorithm:
     * use config['apksigner'] if set
     * try to find apksigner in path
     * find apksigner in build-tools starting from newest installed
       going down to MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION
     :return: path to apksigner or None if no version is found
+
     """
-    if set_command_in_config('apksigner'):
+    command = 'apksigner'
-        return config['apksigner']
+    if command in config:
-    build_tools_path = os.path.join(config['sdk_path'], 'build-tools')
+        return
+
+    tmp = find_command(command)
+    if tmp is not None:
+        config[command] = tmp
+        return
+
+    build_tools_path = os.path.join(config.get('sdk_path', ''), 'build-tools')
     if not os.path.isdir(build_tools_path):
-        return None
+        return
     for f in sorted(os.listdir(build_tools_path), reverse=True):
         if not os.path.isdir(os.path.join(build_tools_path, f)):
             continue
         try:
             if LooseVersion(f) < LooseVersion(MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION):
-                return None
+                return
         except TypeError:
             continue
         if os.path.exists(os.path.join(build_tools_path, f, 'apksigner')):
@@ -483,7 +497,6 @@ def find_apksigner():
             logging.info("Using %s " % apksigner)
             # memoize result
             config['apksigner'] = apksigner
-            return config['apksigner']
 
 
 def find_sdk_tools_cmd(cmd):
@@ -3001,7 +3014,7 @@ def _zipalign(unsigned_apk, aligned_apk):
 
 
 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
-    """Implats a signature from metadata into an APK.
+    """Implants a signature from metadata into an APK.
 
     Note: this changes there supplied APK in place. So copy it if you
     need the original to be preserved.
@@ -3061,82 +3074,48 @@ def get_min_sdk_version(apk):
 def sign_apk(unsigned_path, signed_path, keyalias):
     """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
 
-    Use apksigner for making v2 and v3 signature for apks with targetSDK >=30 as
+    NONE is a Java keyword used to configure smartcards as the
-    otherwise they won't be installable on Android 11/R.
+    keystore.  Otherwise, the keystore is a local file.
+    https://docs.oracle.com/javase/7/docs/technotes/guides/security/p11guide.html#KeyToolJarSigner
 
-    Otherwise use jarsigner for v1 only signatures until we have apksig v2/v3
+    When using smartcards, apksigner does not use the same options has
-    signature transplantig support.
+    Java/keytool/jarsigner (-providerName, -providerClass,
-
+    -providerArg, -storetype).  apksigner documents the options as
-    When using jarsigner we need to manually select the hash algorithm,
+    --ks-provider-class and --ks-provider-arg.  Those seem to be
-    apksigner does this automatically. Apksigner also does the zipalign for us.
+    accepted but fail when actually making a signature with weird
-
+    internal exceptions. We use the options that actually work.  From:
-    SHA-256 support was added in android-18 (4.3), before then, the only options were MD5
+    https://geoffreymetais.github.io/code/key-signing/#scripting
-    and SHA1. This aims to use SHA-256 when the APK does not target
-    older Android versions, and is therefore safe to do so.
-
-    https://issuetracker.google.com/issues/36956587
-    https://android-review.googlesource.com/c/platform/libcore/+/44491
 
     """
-    apk = _get_androguard_APK(unsigned_path)
+    if config['keystore'] == 'NONE':
-    if apk.get_effective_target_sdk_version() >= 30:
+        apksigner_smartcardoptions = config['smartcardoptions'].copy()
-        if config['keystore'] == 'NONE':
+        if '-providerName' in apksigner_smartcardoptions:
-            # NOTE: apksigner doesn't like -providerName/--provider-name at all, don't use that.
+            pos = config['smartcardoptions'].index('-providerName')
-            #       apksigner documents the options as --ks-provider-class and --ks-provider-arg
+            # remove -providerName and it's argument
-            #       those seem to be accepted but fail when actually making a signature with
+            del apksigner_smartcardoptions[pos]
-            #       weird internal exceptions. Those options actually work.
+            del apksigner_smartcardoptions[pos]
-            #       From: https://geoffreymetais.github.io/code/key-signing/#scripting
+        replacements = {'-storetype': '--ks-type',
-            apksigner_smartcardoptions = config['smartcardoptions'].copy()
+                        '-providerClass': '--provider-class',
-            if '-providerName' in apksigner_smartcardoptions:
+                        '-providerArg': '--provider-arg'}
-                pos = config['smartcardoptions'].index('-providerName')
+        signing_args = [replacements.get(n, n) for n in apksigner_smartcardoptions]
-                # remove -providerName and it's argument
-                del apksigner_smartcardoptions[pos]
-                del apksigner_smartcardoptions[pos]
-            replacements = {'-storetype': '--ks-type',
-                            '-providerClass': '--provider-class',
-                            '-providerArg': '--provider-arg'}
-            signing_args = [replacements.get(n, n) for n in apksigner_smartcardoptions]
-        else:
-            signing_args = ['--key-pass', 'env:FDROID_KEY_PASS']
-        if not find_apksigner():
-            raise BuildException(_("apksigner not found, it's required for signing!"))
-        cmd = [find_apksigner(), 'sign',
-               '--ks', config['keystore'],
-               '--ks-pass', 'env:FDROID_KEY_STORE_PASS']
-        cmd += signing_args
-        cmd += ['--ks-key-alias', keyalias,
-                '--in', unsigned_path,
-                '--out', signed_path]
-        p = FDroidPopen(cmd, envs={
-            'FDROID_KEY_STORE_PASS': config['keystorepass'],
-            'FDROID_KEY_PASS': config.get('keypass', "")})
-        if p.returncode != 0:
-            raise BuildException(_("Failed to sign application"), p.output)
-        os.remove(unsigned_path)
     else:
-
+        signing_args = ['--key-pass', 'env:FDROID_KEY_PASS']
-        if get_min_sdk_version(apk) < 18:
+    apksigner = config.get('apksigner', '')
-            signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
+    if not shutil.which(apksigner):
-        else:
+        raise BuildException(_("apksigner not found, it's required for signing!"))
-            signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
+    cmd = [apksigner, 'sign',
-        if config['keystore'] == 'NONE':
+           '--ks', config['keystore'],
-            signing_args = config['smartcardoptions']
+           '--ks-pass', 'env:FDROID_KEY_STORE_PASS']
-        else:
+    cmd += signing_args
-            signing_args = ['-keypass:env', 'FDROID_KEY_PASS']
+    cmd += ['--ks-key-alias', keyalias,
-
+            '--in', unsigned_path,
-        cmd = [config['jarsigner'], '-keystore', config['keystore'],
+            '--out', signed_path]
-               '-storepass:env', 'FDROID_KEY_STORE_PASS']
+    p = FDroidPopen(cmd, envs={
-        cmd += signing_args
+        'FDROID_KEY_STORE_PASS': config['keystorepass'],
-        cmd += signature_algorithm
+        'FDROID_KEY_PASS': config.get('keypass', "")})
-        cmd += [unsigned_path, keyalias]
+    if p.returncode != 0:
-        p = FDroidPopen(cmd, envs={
+        raise BuildException(_("Failed to sign application"), p.output)
-            'FDROID_KEY_STORE_PASS': config['keystorepass'],
+    os.remove(unsigned_path)
-            'FDROID_KEY_PASS': config.get('keypass', "")})
-        if p.returncode != 0:
-            raise BuildException(_("Failed to sign application"), p.output)
-
-        _zipalign(unsigned_path, signed_path)
-        os.remove(unsigned_path)
 
 
 def verify_apks(signed_apk, unsigned_apk, tmp_dir):

fdroidserver/exception.py

@@ -1,5 +1,4 @@
 class FDroidException(Exception):
-
     def __init__(self, value=None, detail=None):
         self.value = value
         self.detail = detail
@@ -22,12 +21,14 @@ class FDroidException(Exception):
         else:
             ret = str(self.value)
         if self.detail:
-            ret += "\n==== detail begin ====\n%s\n==== detail end ====" % ''.join(self.detail).strip()
+            ret += (
+                "\n==== detail begin ====\n%s\n==== detail end ===="
+                % ''.join(self.detail).strip()
+            )
         return ret
 
 
 class MetaDataException(Exception):
-
     def __init__(self, value):
         self.value = value
 

fdroidserver/gpgsign.py

@@ -73,9 +73,7 @@ def main():
             sigpath = os.path.join(output_dir, sigfilename)
 
             if not os.path.exists(sigpath):
-                gpgargs = ['gpg', '-a',
+                gpgargs = ['gpg', '-a', '--output', sigpath, '--detach-sig']
-                           '--output', sigpath,
-                           '--detach-sig']
                 if 'gpghome' in config:
                     gpgargs.extend(['--homedir', config['gpghome']])
                 if 'gpgkey' in config:

fdroidserver/init.py

@@ -146,12 +146,6 @@ def main():
     # now that we have a local config.yml, read configuration...
     config = common.read_config(options)
 
-    # enable apksigner by default so v2/v3 APK signatures validate
-    if common.find_apksigner() is not None:
-        apksigner = common.find_apksigner()
-        test_config['apksigner'] = apksigner
-        common.write_to_config(test_config, 'apksigner', apksigner)
-
     # the NDK is optional and there may be multiple versions of it, so it's
     # left for the user to configure
 

fdroidserver/publish.py

@@ -146,6 +146,9 @@ def status_update_json(generatedKeys, signedApks):
 
     logging.debug(_('Outputting JSON'))
     output = common.setup_status_output(start_timestamp)
+    output['apksigner'] = shutil.which(config.get('apksigner', ''))
+    output['jarsigner'] = shutil.which(config.get('jarsigner', ''))
+    output['keytool'] = shutil.which(config.get('keytool', ''))
     if generatedKeys:
         output['generatedKeys'] = generatedKeys
     if signedApks:

fdroidserver/signindex.py

@@ -99,7 +99,8 @@ def main():
 
     if 'jarsigner' not in config:
         raise FDroidException(
-            _('Java jarsigner not found! Install in standard location or set java_paths!'))
+            _('Java jarsigner not found! Install in standard location or set java_paths!')
+        )
 
     repodirs = ['repo']
     if config['archive_older'] != 0:

fdroidserver/tail.py

@@ -33,6 +33,7 @@ import threading
 
 class Tail(object):
     ''' Represents a tail command. '''
+
     def __init__(self, tailed_file):
         ''' Initiate a Tail instance.
             Check for file validity, assigns callback function to standard out.
@@ -95,7 +96,6 @@ class Tail(object):
 
 
 class TailError(Exception):
-
     def __init__(self, msg):
         self.message = msg
 

fdroidserver/update.py

@@ -144,6 +144,9 @@ def status_update_json(apps, apks):
     output['noPackages'] = []
     output['needsUpdate'] = []
     output['noUpdateCheck'] = []
+    output['apksigner'] = shutil.which(config.get('apksigner', ''))
+    output['jarsigner'] = shutil.which(config.get('jarsigner', ''))
+    output['keytool'] = shutil.which(config.get('keytool', ''))
 
     for appid in apps:
         app = apps[appid]
@@ -1395,17 +1398,17 @@ def scan_apk(apk_file):
     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
     apk['sig'] = getsig(apk_file)
     if not apk['sig']:
-        raise BuildException("Failed to get apk signature")
+        raise BuildException(_("Failed to get APK signing key fingerprint"))
     apk['signer'] = common.apk_signer_fingerprint(os.path.join(os.getcwd(),
                                                                apk_file))
     if not apk.get('signer'):
-        raise BuildException("Failed to get apk signing key fingerprint")
+        raise BuildException(_("Failed to get APK signing key fingerprint"))
 
     # Get size of the APK
     apk['size'] = os.path.getsize(apk_file)
 
     if 'minSdkVersion' not in apk:
-        logging.warning("No SDK version information found in {0}".format(apk_file))
+        logging.warning(_("No minimum SDK version found in {0}, using default (3).").format(apk_file))
         apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
 
     # Check for known vulnerabilities
@@ -1524,13 +1527,16 @@ def scan_apk_androguard(apk, apkfile):
     icon_id_str = apkobject.get_element("application", "icon")
     if icon_id_str:
         icon_id = int(icon_id_str.replace("@", "0x"), 16)
-        resource_id = arsc.get_id(apk['packageName'], icon_id)
+        try:
-        if resource_id:
+            resource_id = arsc.get_id(apk['packageName'], icon_id)
-            icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
+            if resource_id:
-        else:
+                icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
-            # don't use 'anydpi' aka 0xFFFE aka 65534 since it is XML
+            else:
-            icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon(max_dpi=65534 - 1)))[0]
+                # don't use 'anydpi' aka 0xFFFE aka 65534 since it is XML
-        apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
+                icon_name = os.path.splitext(os.path.basename(apkobject.get_app_icon(max_dpi=65534 - 1)))[0]
+            apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)
+        except Exception as e:
+            logging.error("Cannot fetch icon from %s: %s" % (apkfile, str(e)))
 
     arch_re = re.compile("^lib/(.*)/.*$")
     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])

tests/build.TestCase

@@ -37,6 +37,8 @@ class BuildTest(unittest.TestCase):
         if not os.path.exists(self.tmpdir):
             os.makedirs(self.tmpdir)
         os.chdir(self.basedir)
+        fdroidserver.common.config = None
+        fdroidserver.build.config = None
 
     def test_get_apk_metadata(self):
         config = dict()

tests/common.TestCase

@@ -619,12 +619,9 @@ class CommonTest(unittest.TestCase):
     def test_sign_apk(self):
         fdroidserver.common.config = None
         config = fdroidserver.common.read_config(fdroidserver.common.options)
-        try:
+        if 'apksigner' not in config:
-            fdroidserver.common.find_sdk_tools_cmd('zipalign')
+            self.skipTest('SKIPPING test_sign_apk, apksigner not installed!')
-        except fdroidserver.exception.FDroidException:
-            self.skipTest('SKIPPING test_sign_apk, zipalign not installed!')
 
-        config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
         config['keyalias'] = 'sova'
         config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
         config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
@@ -656,18 +653,6 @@ class CommonTest(unittest.TestCase):
         self.assertTrue(fdroidserver.common.verify_apk_signature(signed))
         self.assertEqual('18', fdroidserver.common._get_androguard_APK(signed).get_min_sdk_version())
 
-    def test_sign_apk_targetsdk_30(self):
-        fdroidserver.common.config = None
-        config = fdroidserver.common.read_config(fdroidserver.common.options)
-        if not fdroidserver.common.find_apksigner():
-            self.skipTest('SKIPPING as apksigner is not installed!')
-        config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
-        config['keyalias'] = 'sova'
-        config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
-        config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
-        config['keystore'] = os.path.join(self.basedir, 'keystore.jks')
-        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
-
         shutil.copy(os.path.join(self.basedir, 'minimal_targetsdk_30_unsigned.apk'), testdir)
         unsigned = os.path.join(testdir, 'minimal_targetsdk_30_unsigned.apk')
         signed = os.path.join(testdir, 'minimal_targetsdk_30.apk')
@@ -681,18 +666,6 @@ class CommonTest(unittest.TestCase):
         # verify it has a v2 signature
         self.assertTrue(fdroidserver.common._get_androguard_APK(signed).is_signed_v2())
 
-    def test_sign_no_targetsdk(self):
-        fdroidserver.common.config = None
-        config = fdroidserver.common.read_config(fdroidserver.common.options)
-        if not fdroidserver.common.find_apksigner():
-            self.skipTest('SKIPPING as apksigner is not installed!')
-        config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
-        config['keyalias'] = 'sova'
-        config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
-        config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
-        config['keystore'] = os.path.join(self.basedir, 'keystore.jks')
-        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
-
         shutil.copy(os.path.join(self.basedir, 'no_targetsdk_minsdk30_unsigned.apk'), testdir)
         unsigned = os.path.join(testdir, 'no_targetsdk_minsdk30_unsigned.apk')
         signed = os.path.join(testdir, 'no_targetsdk_minsdk30_signed.apk')
@@ -1442,7 +1415,7 @@ class CommonTest(unittest.TestCase):
         self.assertFalse(os.path.exists('config.yml'))
         self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config(fdroidserver.common.options)
-        self.assertEqual(None, config.get('apksigner'))
+        self.assertIsNone(config.get('stats_server'))
         self.assertIsNotNone(config.get('char_limits'))
 
     def test_with_config_yml(self):

tests/complete-ci-tests

@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# this is the script run by the Jenkins and gitlab-ci continuous integration
-# build services.  It is a thorough set of tests that runs all the tests using
-# the various methods of installing/running fdroidserver.  It is separate from
-# ./tests/run-tests because its too heavy for manual use.
-
-if [ `dirname $0` != "." ]; then
-    echo "only run this script like ./`basename $0`"
-    exit 1
-fi
-
-set -e
-set -x
-
-if [ -z $WORKSPACE ]; then
-    WORKSPACE=`cd $(dirname $0)/.. && pwd`
-    echo "Setting Workspace to $WORKSPACE"
-fi
-
-if [ -z $ANDROID_HOME ]; then
-    if [ -e ~/.android/bashrc ]; then
-        . ~/.android/bashrc
-    else
-        echo "ANDROID_HOME must be set!"
-        exit 1
-    fi
-fi
-
-if ! which pyvenv; then
-    echo "pyvenv required to run this test suite!"
-    exit 1
-fi
-
-apksource=$1
-
-#------------------------------------------------------------------------------#
-# cache pypi downloads
-if [ -z $PIP_DOWNLOAD_CACHE ]; then
-    export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache
-fi
-
-#------------------------------------------------------------------------------#
-# run local tests, don't scan fdroidserver/ project for APKs
-
-cd $WORKSPACE/tests
-./run-tests $apksource
-
-
-#------------------------------------------------------------------------------#
-# make sure that translations do not cause stacktraces
-cd $WORKSPACE/locale
-for locale in *; do
-    if [ ! -d $locale ]; then
-        continue
-    fi
-    for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $WORKSPACE/fdroid`; do
-        LANGUAGE=$locale $WORKSPACE/fdroid $cmd --help > /dev/null
-    done
-done
-
-
-#------------------------------------------------------------------------------#
-# test install using install direct from git repo
-cd $WORKSPACE
-rm -rf $WORKSPACE/env
-pyvenv $WORKSPACE/env
-. $WORKSPACE/env/bin/activate
-pip3 install --quiet -e $WORKSPACE[test]
-python3 setup.py compile_catalog install
-
-# make sure translation files were installed
-test -e $WORKSPACE/env/share/locale/de/LC_MESSAGES/fdroidserver.mo
-
-# run tests in new pip+pyvenv install
-fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource

tests/init.TestCase

@@ -10,7 +10,6 @@ import shutil
 import sys
 import tempfile
 import unittest
-import yaml
 
 
 localmodule = os.path.realpath(
@@ -68,11 +67,7 @@ class InitTest(unittest.TestCase):
 
         sys.argv = ['fdroid init', '--keystore', 'keystore.jks', '--repo-keyalias=sova']
         fdroidserver.init.main()
-        with open('config.yml') as fp:
+        self.assertEqual(apksigner, fdroidserver.init.config.get('apksigner'))
-            config = yaml.safe_load(fp)
-        self.assertTrue(os.path.exists(config['keystore']))
-        self.assertTrue(os.path.exists(config['apksigner']))
-        self.assertEqual(apksigner, config['apksigner'])
 
 
 if __name__ == "__main__":

tests/publish.TestCase

@@ -11,6 +11,7 @@
 #
 
 import inspect
+import json
 import logging
 import optparse
 import os
@@ -221,6 +222,34 @@ class PublishTest(unittest.TestCase):
             self.assertTrue(pk.is_decrypted())
             self.assertEqual(jks.util.RSA_ENCRYPTION_OID, pk.algorithm_oid)
 
+    def test_status_update_json(self):
+        common.config = {}
+        publish.config = {}
+        with tempfile.TemporaryDirectory() as tmpdir:
+            os.chdir(tmpdir)
+            with mock.patch('sys.argv', ['fdroid publish', '']):
+                publish.status_update_json([], [])
+                with open('repo/status/publish.json') as fp:
+                    data = json.load(fp)
+                self.assertTrue('apksigner' in data)
+
+                publish.config = {
+                    'apksigner': 'apksigner',
+                }
+                publish.status_update_json([], [])
+                with open('repo/status/publish.json') as fp:
+                    data = json.load(fp)
+                self.assertEqual(shutil.which(publish.config['apksigner']), data['apksigner'])
+
+                publish.config = {}
+                common.fill_config_defaults(publish.config)
+                publish.status_update_json([], [])
+                with open('repo/status/publish.json') as fp:
+                    data = json.load(fp)
+                self.assertEqual(publish.config.get('apksigner'), data['apksigner'])
+                self.assertEqual(publish.config['jarsigner'], data['jarsigner'])
+                self.assertEqual(publish.config['keytool'], data['keytool'])
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/run-tests

@@ -72,11 +72,17 @@ have_git_2_3() {
 
 is_MD5_disabled() {
     javac $WORKSPACE/tests/IsMD5Disabled.java && java -cp $WORKSPACE/tests IsMD5Disabled
-    return $?
 }
 
 use_apksigner() {
-    test -x "`sed -En 's,^ *apksigner: +,,p' config.yml`"
+    python3 -c "
+import sys
+sys.path.insert(0, '$WORKSPACE')
+from fdroidserver import common
+c = {'sdk_path': '$ANDROID_HOME'}
+common.find_apksigner(c)
+exit(c.get('apksigner') is None)
+"
 }
 
 #------------------------------------------------------------------------------#
@@ -169,7 +175,7 @@ $fdroid --version
 #------------------------------------------------------------------------------#
 echo_header 'run process when building and signing are on separate machines'
 
-if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then
+if use_apksigner; then
     REPOROOT=`create_test_dir`
     cd $REPOROOT
     cp $WORKSPACE/tests/keystore.jks $REPOROOT/
@@ -213,20 +219,6 @@ $fdroid readmeta
 $fdroid update
 
 
-#------------------------------------------------------------------------------#
-echo_header 'run "fdroid build" in fresh git checkout from import.TestCase'
-
-cd $WORKSPACE/tests/tmp/importer
-git remote update -p
-git clean -fdx
-# stick with known working commit, in case future commits break things for this code
-git reset --hard fea54e1161d5eb9eb1a54e26253ef84d3ab63705
-if [ -d $ANDROID_HOME/platforms/android-23 && -d $ANDROID_HOME/build-tools/23.0.3 ]; then
-    $fdroid build --verbose org.fdroid.ci.test.app:300
-else
-    echo 'WARNING: Skipping "fdroid build" test since android-23 is missing!'
-fi
-
 #------------------------------------------------------------------------------#
 echo_header 'copy git import and run "fdroid scanner" on it'
 
@@ -325,8 +317,9 @@ else
     test `grep '<package>' repo/index.xml | wc -l` -eq 7
 fi
 
+
 #------------------------------------------------------------------------------#
-if ! which apksigner; then
+if ! use_apksigner; then
     echo_header 'test per-app "Archive Policy"'
 
     REPOROOT=`create_test_dir`
@@ -534,7 +527,7 @@ test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk
 test -e archive/urzip-badsig.apk
 
 sed -i.tmp '/apksigner:/d' config.yml
-if ! which apksigner; then
+if ! use_apksigner; then
     $sed -i.tmp '/allow_disabled_algorithms/d' config.yml
     $fdroid update --pretty --nosign
     test `grep '<package>' archive/index.xml | wc -l` -eq 5
@@ -1240,28 +1233,30 @@ fi
 #------------------------------------------------------------------------------#
 echo_header 'test extracting and publishing with developer signature'
 
-REPOROOT=`create_test_dir`
+if use_apksigner; then
-cd $REPOROOT
+    REPOROOT=`create_test_dir`
-fdroid_init_with_prebuilt_keystore
+    cd $REPOROOT
-echo 'keydname: "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.yml
+    fdroid_init_with_prebuilt_keystore
-test -d metadata || mkdir metadata
+    echo 'keydname: "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.yml
-cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/
+    test -d metadata || mkdir metadata
-test -d repo || mkdir repo
+    cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/
-test -d unsigned || mkdir unsigned
+    test -d repo || mkdir repo
-cp $WORKSPACE/tests/repo/com.politedroid_6.apk unsigned/
+    test -d unsigned || mkdir unsigned
-$fdroid signatures unsigned/com.politedroid_6.apk
+    cp $WORKSPACE/tests/repo/com.politedroid_6.apk unsigned/
-test -d metadata/com.politedroid/signatures/6
+    $fdroid signatures unsigned/com.politedroid_6.apk
-test -f metadata/com.politedroid/signatures/6/MANIFEST.MF
+    test -d metadata/com.politedroid/signatures/6
-test -f metadata/com.politedroid/signatures/6/RELEASE.RSA
+    test -f metadata/com.politedroid/signatures/6/MANIFEST.MF
-test -f metadata/com.politedroid/signatures/6/RELEASE.SF
+    test -f metadata/com.politedroid/signatures/6/RELEASE.RSA
-! test -f repo/com.politedroid_6.apk
+    test -f metadata/com.politedroid/signatures/6/RELEASE.SF
-$fdroid publish
+    ! test -f repo/com.politedroid_6.apk
-test -f repo/com.politedroid_6.apk
+    $fdroid publish
-if which jarsigner; then
+    test -f repo/com.politedroid_6.apk
-    jarsigner -verify repo/com.politedroid_6.apk
+    if which apksigner; then
-fi
+	apksigner verify repo/com.politedroid_6.apk
-if which apksigner; then
+    fi
-    apksigner verify repo/com.politedroid_6.apk
+    if which jarsigner; then
+	jarsigner -verify repo/com.politedroid_6.apk
+    fi
 fi
 
 

tests/update.TestCase

@@ -6,6 +6,7 @@ import copy
 import git
 import glob
 import inspect
+import json
 import logging
 import optparse
 import os
@@ -23,6 +24,7 @@ from binascii import unhexlify
 from datetime import datetime
 from distutils.version import LooseVersion
 from testcommon import TmpCwd
+from unittest import mock
 
 try:
     from yaml import CSafeLoader as SafeLoader
@@ -63,6 +65,7 @@ DONATION_FIELDS = (
 class Options:
     allow_disabled_algorithms = False
     clean = False
+    pretty = True
     rename_apks = False
 
 
@@ -656,9 +659,7 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.config = config
         os.chdir(os.path.join(localmodule, 'tests'))
 
-        apksigner = fdroidserver.common.find_apksigner()
+        if 'apksigner' in config:
-        if apksigner:
-            config['apksigner'] = apksigner
             apk_info = fdroidserver.update.scan_apk('v2.only.sig_2.apk')
             self.assertIsNone(apk_info.get('maxSdkVersion'))
             self.assertEqual(apk_info.get('versionName'), 'v2-only')
@@ -1376,6 +1377,35 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update._set_author_entry(app, key, f)
         self.assertIsNone(app.get(key))
 
+    def test_status_update_json(self):
+        fdroidserver.common.config = {}
+        fdroidserver.update.config = {}
+        fdroidserver.update.options = Options
+        with tempfile.TemporaryDirectory() as tmpdir:
+            os.chdir(tmpdir)
+            with mock.patch('sys.argv', ['fdroid update', '']):
+                fdroidserver.update.status_update_json([], [])
+                with open('repo/status/update.json') as fp:
+                    data = json.load(fp)
+                self.assertTrue('apksigner' in data)
+
+                fdroidserver.update.config = {
+                    'apksigner': 'apksigner',
+                }
+                fdroidserver.update.status_update_json([], [])
+                with open('repo/status/update.json') as fp:
+                    data = json.load(fp)
+                self.assertEqual(shutil.which(fdroidserver.update.config['apksigner']), data['apksigner'])
+
+                fdroidserver.update.config = {}
+                fdroidserver.common.fill_config_defaults(fdroidserver.update.config)
+                fdroidserver.update.status_update_json([], [])
+                with open('repo/status/update.json') as fp:
+                    data = json.load(fp)
+                self.assertEqual(fdroidserver.update.config.get('apksigner'), data['apksigner'])
+                self.assertEqual(fdroidserver.update.config['jarsigner'], data['jarsigner'])
+                self.assertEqual(fdroidserver.update.config['keytool'], data['keytool'])
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))