Merge branch 'scanner_dexdump' into 'master'

[scanner] replace apkanalyzer by dexdump

See merge request fdroid/fdroidserver!1110

.gitlab-ci.yml

@@ -122,7 +122,7 @@ ubuntu_bionic_pip:
   image: ubuntu:bionic
   <<: *apt-template
   script:
-    - apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13
+    - apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13 dexdump
     - rm -rf env
     - pyvenv env
     - . env/bin/activate

MANIFEST.in

@@ -42,6 +42,7 @@ include locale/zh_Hant/LC_MESSAGES/fdroidserver.po
 include makebuildserver
 include README.md
 include tests/androguard_test.py
+include tests/apk.embedded_1.apk
 include tests/bad-unicode-*.apk
 include tests/build.TestCase
 include tests/build-tools/17.0.0/aapt-output-com.moez.QKSMS_182.txt

fdroidserver/common.py

@@ -585,6 +585,9 @@ def find_sdk_tools_cmd(cmd):
         sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
         if os.path.exists(sdk_platform_tools):
             tooldirs.append(sdk_platform_tools)
+    sdk_build_tools = glob.glob(os.path.join(config['sdk_path'], 'build-tools', '*.*'))
+    if sdk_build_tools:
+        tooldirs.append(sorted(sdk_build_tools)[-1])  # use most recent version
     if os.path.exists('/usr/bin'):
         tooldirs.append('/usr/bin')
     for d in tooldirs:

fdroidserver/scanner.py

@@ -22,8 +22,10 @@ import os
 import re
 import sys
 import traceback
+import zipfile
 from argparse import ArgumentParser
 from copy import deepcopy
+from tempfile import TemporaryDirectory
 import logging
 import itertools
 
@@ -42,19 +44,13 @@ MAVEN_URL_REGEX = re.compile(r"""\smaven\s*{.*?(?:setUrl|url)\s*=?\s*(?:uri)?\(?
                              re.DOTALL)
 
 CODE_SIGNATURES = {
-    # The `apkanalyzer dex packages` output looks like this:
-    # M d 1   1       93      <packagename> <other stuff>
-    # The first column has P/C/M/F for package, class, method or field
-    # The second column has x/k/r/d for removed, kept, referenced and defined.
-    # We already filter for defined only in the apkanalyzer call. 'r' will be
-    # for things referenced but not distributed in the apk.
-    exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [
-        r'(com\.google\.firebase[^\s]*)',
-        r'(com\.google\.android\.gms[^\s]*)',
-        r'(com\.google\.android\.play\.core[^\s]*)',
-        r'(com\.google\.tagmanager[^\s]*)',
-        r'(com\.google\.analytics[^\s]*)',
-        r'(com\.android\.billing[^\s]*)',
+    exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
+        r'com/google/firebase',
+        r'com/google/android/gms',
+        r'com/google/android/play/core',
+        r'com/google/tagmanager',
+        r'com/google/analytics',
+        r'com/android/billing',
     ]
 }
 
@@ -106,26 +102,47 @@ def get_gradle_compile_commands(build):
     return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands]
 
 
-def scan_binary(apkfile):
-    """Scan output of apkanalyzer for known non-free classes.
-
-    apkanalyzer produces useful output when it can run, but it does
-    not support all recent JDK versions, and also some DEX versions,
-    so this cannot count on it to always produce useful output or even
-    to run without exiting with an error.
-
+def get_embedded_classes(apkfile, depth=0):
     """
-    logging.info(_('Scanning APK with apkanalyzer for known non-free classes.'))
-    result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False)
-    if result.returncode != 0:
-        logging.warning(_('scanner not cleanly run apkanalyzer: %s') % result.output)
+    Get the list of Java classes embedded into all DEX files.
+
+    :return: set of Java classes names as string
+    """
+    if depth > 10:  # zipbomb protection
+        return {_('Max recursion depth in ZIP file reached: %s') % apkfile}
+
+    apk_regex = re.compile(r'.*\.apk')
+    class_regex = re.compile(r'classes.*\.dex')
+    classes = set()
+
+    try:
+        with TemporaryDirectory() as tmp_dir, zipfile.ZipFile(apkfile, 'r') as apk_zip:
+            for info in apk_zip.infolist():
+                # apk files can contain apk files, again
+                if apk_regex.search(info.filename):
+                    with apk_zip.open(info) as apk_fp:
+                        classes = classes.union(get_embedded_classes(apk_fp, depth + 1))
+
+                elif class_regex.search(info.filename):
+                    apk_zip.extract(info, tmp_dir)
+                    run = common.SdkToolsPopen(["dexdump", '{}/{}'.format(tmp_dir, info.filename)])
+                    classes = classes.union(set(re.findall(r'[A-Z]+((?:\w+\/)+\w+)', run.output)))
+    except zipfile.BadZipFile as ex:
+        return {_('Problem with ZIP file: %s, error %s') % (apkfile, ex)}
+
+    return classes
+
+
+def scan_binary(apkfile):
+    """Scan output of dexdump for known non-free classes."""
+    logging.info(_('Scanning APK with dexdump for known non-free classes.'))
+    result = get_embedded_classes(apkfile)
     problems = 0
-    for suspect, regexp in CODE_SIGNATURES.items():
-        matches = regexp.findall(result.output)
-        if matches:
-            for m in set(matches):
-                logging.debug("Found class '%s'" % m)
-            problems += 1
+    for classname in result:
+        for suspect, regexp in CODE_SIGNATURES.items():
+            if regexp.match(classname):
+                logging.debug("Found class '%s'" % classname)
+                problems += 1
     if problems:
         logging.critical("Found problems in %s" % apkfile)
     return problems

tests/scanner.TestCase

@@ -5,6 +5,7 @@ import inspect
 import logging
 import optparse
 import os
+import re
 import shutil
 import sys
 import tempfile
@@ -202,6 +203,34 @@ class ScannerTest(unittest.TestCase):
             self.assertTrue(f in files['infos'],
                             f + ' should be removed with an info message')
 
+    def test_scan_binary(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.verbose = False
+
+        apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk')
+        self.assertEqual(
+            0,
+            fdroidserver.scanner.scan_binary(apkfile),
+            'Found false positives in binary',
+        )
+        fdroidserver.scanner.CODE_SIGNATURES["java/lang/Object"] = re.compile(
+            r'.*java/lang/Object', re.IGNORECASE | re.UNICODE
+        )
+        self.assertEqual(
+            1,
+            fdroidserver.scanner.scan_binary(apkfile),
+            'Did not find bad code signature in binary',
+        )
+        apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk')
+        self.assertEqual(
+            1,
+            fdroidserver.scanner.scan_binary(apkfile),
+            'Did not find bad code signature in binary',
+        )
+
     def test_build_local_scanner(self):
         """`fdroid build` calls scanner functions, test them here"""
         testdir = tempfile.mkdtemp(