Merge branch 'exodus_scanner' into 'master'

Integrate Exodus

Closes #806, #1008, and #566

See merge request fdroid/fdroidserver!1137

fdroidserver/scanner.py

@@ -24,11 +24,14 @@ import sys
 import traceback
 import zipfile
 from argparse import ArgumentParser
+from collections import namedtuple
 from copy import deepcopy
 from tempfile import TemporaryDirectory
 import logging
 import itertools
 
+import requests
+
 from . import _
 from . import common
 from . import metadata
@@ -140,7 +143,45 @@ def get_embedded_classes(apkfile, depth=0):
     return classes
 
 
-def scan_binary(apkfile):
+# taken from exodus_core
+def _compile_signatures(signatures):
+    """
+    Compiles the regex associated to each signature, in order to speed up the trackers detection.
+
+    :return: A compiled list of signatures.
+    """
+    compiled_tracker_signature = []
+    try:
+        compiled_tracker_signature = [
+            re.compile(track.code_signature) for track in signatures
+        ]
+    except TypeError:
+        print("signatures is not iterable")
+    return compiled_tracker_signature
+
+
+# taken from exodus_core
+def load_trackers_signatures():
+    """
+    Load trackers signatures from the official Exodus database.
+
+    :return: a dictionary containing signatures.
+    """
+    signatures = []
+    exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers"
+    r = requests.get(exodus_url)
+    data = r.json()
+    for e in data['trackers']:
+        signatures.append(
+            namedtuple('tracker', data['trackers'][e].keys())(
+                *data['trackers'][e].values()
+            )
+        )
+    logging.debug('{} trackers signatures loaded'.format(len(signatures)))
+    return signatures, _compile_signatures(signatures)
+
+
+def scan_binary(apkfile, extract_signatures=None):
     """Scan output of dexdump for known non-free classes."""
     logging.info(_('Scanning APK with dexdump for known non-free classes.'))
     result = get_embedded_classes(apkfile)
@@ -150,6 +191,29 @@ def scan_binary(apkfile):
             if regexp.match(classname):
                 logging.debug("Found class '%s'" % classname)
                 problems += 1
+
+    if extract_signatures:
+
+        def _detect_tracker(sig, tracker, class_list):
+            for clazz in class_list:
+                if sig.search(clazz):
+                    return tracker
+            return None
+
+        results = []
+        args = [(extract_signatures[1][index], tracker, result)
+                for (index, tracker) in enumerate(extract_signatures[0]) if
+                len(tracker.code_signature) > 3]
+
+        for res in itertools.starmap(_detect_tracker, args):
+            if res:
+                results.append(res)
+
+        trackers = [t for t in results if t is not None]
+        for tracker in trackers:
+            logging.debug("Found tracker {}".format(tracker.code_signature))
+        problems += len(trackers)
+
     if problems:
         logging.critical("Found problems in %s" % apkfile)
     return problems
@@ -449,9 +513,16 @@ def main():
     global config, options, json_per_build
 
     # Parse command line...
-    parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
+    parser = ArgumentParser(
+        usage="%(prog)s [options] [APPID[:VERCODE] path/to.apk ...]"
+    )
     common.setup_global_opts(parser)
     parser.add_argument("appid", nargs='*', help=_("application ID with optional versionCode in the form APPID[:VERCODE]"))
+    parser.add_argument(
+        "--exodus",
+        action="store_true",
+        help="Use tracker scanner from Exodus project (requires internet)",
+    )
     parser.add_argument("-f", "--force", action="store_true", default=False,
                         help=_("Force scan of disabled apps and builds."))
     parser.add_argument("--json", action="store_true", default=False,
@@ -469,11 +540,32 @@ def main():
 
     config = common.read_config(options)
 
+    probcount = 0
+
+    exodus = []
+    if options.exodus:
+        exodus = load_trackers_signatures()
+
+    appids = []
+    for apk in options.appid:
+        if os.path.isfile(apk):
+            count = scan_binary(apk, exodus)
+            if count > 0:
+                logging.warning(
+                    _('Scanner found {count} problems in {apk}:').format(
+                        count=count, apk=apk
+                    )
+                )
+                probcount += count
+        else:
+            appids.append(apk)
+
+    if not appids:
+        return
+
     # Read all app and srclib metadata
     allapps = metadata.read_metadata()
-    apps = common.read_app_args(options.appid, allapps, True)
+    apps = common.read_app_args(appids, allapps, True)
-
-    probcount = 0
 
     build_dir = 'build'
     if not os.path.isdir(build_dir):