mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-05 18:50:09 +02:00
Merge branch 'scanner_dexdump' into 'master'
[scanner] replace apkanalyzer by dexdump See merge request fdroid/fdroidserver!1110
This commit is contained in:
commit
6318bf0f5d
@ -122,7 +122,7 @@ ubuntu_bionic_pip:
|
||||
image: ubuntu:bionic
|
||||
<<: *apt-template
|
||||
script:
|
||||
- apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13
|
||||
- apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13 dexdump
|
||||
- rm -rf env
|
||||
- pyvenv env
|
||||
- . env/bin/activate
|
||||
|
@ -42,6 +42,7 @@ include locale/zh_Hant/LC_MESSAGES/fdroidserver.po
|
||||
include makebuildserver
|
||||
include README.md
|
||||
include tests/androguard_test.py
|
||||
include tests/apk.embedded_1.apk
|
||||
include tests/bad-unicode-*.apk
|
||||
include tests/build.TestCase
|
||||
include tests/build-tools/17.0.0/aapt-output-com.moez.QKSMS_182.txt
|
||||
|
@ -585,6 +585,9 @@ def find_sdk_tools_cmd(cmd):
|
||||
sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
|
||||
if os.path.exists(sdk_platform_tools):
|
||||
tooldirs.append(sdk_platform_tools)
|
||||
sdk_build_tools = glob.glob(os.path.join(config['sdk_path'], 'build-tools', '*.*'))
|
||||
if sdk_build_tools:
|
||||
tooldirs.append(sorted(sdk_build_tools)[-1]) # use most recent version
|
||||
if os.path.exists('/usr/bin'):
|
||||
tooldirs.append('/usr/bin')
|
||||
for d in tooldirs:
|
||||
|
@ -22,8 +22,10 @@ import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
import zipfile
|
||||
from argparse import ArgumentParser
|
||||
from copy import deepcopy
|
||||
from tempfile import TemporaryDirectory
|
||||
import logging
|
||||
import itertools
|
||||
|
||||
@ -42,19 +44,13 @@ MAVEN_URL_REGEX = re.compile(r"""\smaven\s*{.*?(?:setUrl|url)\s*=?\s*(?:uri)?\(?
|
||||
re.DOTALL)
|
||||
|
||||
CODE_SIGNATURES = {
|
||||
# The `apkanalyzer dex packages` output looks like this:
|
||||
# M d 1 1 93 <packagename> <other stuff>
|
||||
# The first column has P/C/M/F for package, class, method or field
|
||||
# The second column has x/k/r/d for removed, kept, referenced and defined.
|
||||
# We already filter for defined only in the apkanalyzer call. 'r' will be
|
||||
# for things referenced but not distributed in the apk.
|
||||
exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [
|
||||
r'(com\.google\.firebase[^\s]*)',
|
||||
r'(com\.google\.android\.gms[^\s]*)',
|
||||
r'(com\.google\.android\.play\.core[^\s]*)',
|
||||
r'(com\.google\.tagmanager[^\s]*)',
|
||||
r'(com\.google\.analytics[^\s]*)',
|
||||
r'(com\.android\.billing[^\s]*)',
|
||||
exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
|
||||
r'com/google/firebase',
|
||||
r'com/google/android/gms',
|
||||
r'com/google/android/play/core',
|
||||
r'com/google/tagmanager',
|
||||
r'com/google/analytics',
|
||||
r'com/android/billing',
|
||||
]
|
||||
}
|
||||
|
||||
@ -106,26 +102,47 @@ def get_gradle_compile_commands(build):
|
||||
return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands]
|
||||
|
||||
|
||||
def scan_binary(apkfile):
|
||||
"""Scan output of apkanalyzer for known non-free classes.
|
||||
|
||||
apkanalyzer produces useful output when it can run, but it does
|
||||
not support all recent JDK versions, and also some DEX versions,
|
||||
so this cannot count on it to always produce useful output or even
|
||||
to run without exiting with an error.
|
||||
|
||||
def get_embedded_classes(apkfile, depth=0):
|
||||
"""
|
||||
logging.info(_('Scanning APK with apkanalyzer for known non-free classes.'))
|
||||
result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False)
|
||||
if result.returncode != 0:
|
||||
logging.warning(_('scanner not cleanly run apkanalyzer: %s') % result.output)
|
||||
Get the list of Java classes embedded into all DEX files.
|
||||
|
||||
:return: set of Java classes names as string
|
||||
"""
|
||||
if depth > 10: # zipbomb protection
|
||||
return {_('Max recursion depth in ZIP file reached: %s') % apkfile}
|
||||
|
||||
apk_regex = re.compile(r'.*\.apk')
|
||||
class_regex = re.compile(r'classes.*\.dex')
|
||||
classes = set()
|
||||
|
||||
try:
|
||||
with TemporaryDirectory() as tmp_dir, zipfile.ZipFile(apkfile, 'r') as apk_zip:
|
||||
for info in apk_zip.infolist():
|
||||
# apk files can contain apk files, again
|
||||
if apk_regex.search(info.filename):
|
||||
with apk_zip.open(info) as apk_fp:
|
||||
classes = classes.union(get_embedded_classes(apk_fp, depth + 1))
|
||||
|
||||
elif class_regex.search(info.filename):
|
||||
apk_zip.extract(info, tmp_dir)
|
||||
run = common.SdkToolsPopen(["dexdump", '{}/{}'.format(tmp_dir, info.filename)])
|
||||
classes = classes.union(set(re.findall(r'[A-Z]+((?:\w+\/)+\w+)', run.output)))
|
||||
except zipfile.BadZipFile as ex:
|
||||
return {_('Problem with ZIP file: %s, error %s') % (apkfile, ex)}
|
||||
|
||||
return classes
|
||||
|
||||
|
||||
def scan_binary(apkfile):
|
||||
"""Scan output of dexdump for known non-free classes."""
|
||||
logging.info(_('Scanning APK with dexdump for known non-free classes.'))
|
||||
result = get_embedded_classes(apkfile)
|
||||
problems = 0
|
||||
for suspect, regexp in CODE_SIGNATURES.items():
|
||||
matches = regexp.findall(result.output)
|
||||
if matches:
|
||||
for m in set(matches):
|
||||
logging.debug("Found class '%s'" % m)
|
||||
problems += 1
|
||||
for classname in result:
|
||||
for suspect, regexp in CODE_SIGNATURES.items():
|
||||
if regexp.match(classname):
|
||||
logging.debug("Found class '%s'" % classname)
|
||||
problems += 1
|
||||
if problems:
|
||||
logging.critical("Found problems in %s" % apkfile)
|
||||
return problems
|
||||
|
BIN
tests/apk.embedded_1.apk
Normal file
BIN
tests/apk.embedded_1.apk
Normal file
Binary file not shown.
@ -5,6 +5,7 @@ import inspect
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
@ -202,6 +203,34 @@ class ScannerTest(unittest.TestCase):
|
||||
self.assertTrue(f in files['infos'],
|
||||
f + ' should be removed with an info message')
|
||||
|
||||
def test_scan_binary(self):
|
||||
config = dict()
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.common.config = config
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.verbose = False
|
||||
|
||||
apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk')
|
||||
self.assertEqual(
|
||||
0,
|
||||
fdroidserver.scanner.scan_binary(apkfile),
|
||||
'Found false positives in binary',
|
||||
)
|
||||
fdroidserver.scanner.CODE_SIGNATURES["java/lang/Object"] = re.compile(
|
||||
r'.*java/lang/Object', re.IGNORECASE | re.UNICODE
|
||||
)
|
||||
self.assertEqual(
|
||||
1,
|
||||
fdroidserver.scanner.scan_binary(apkfile),
|
||||
'Did not find bad code signature in binary',
|
||||
)
|
||||
apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk')
|
||||
self.assertEqual(
|
||||
1,
|
||||
fdroidserver.scanner.scan_binary(apkfile),
|
||||
'Did not find bad code signature in binary',
|
||||
)
|
||||
|
||||
def test_build_local_scanner(self):
|
||||
"""`fdroid build` calls scanner functions, test them here"""
|
||||
testdir = tempfile.mkdtemp(
|
||||
|
Loading…
Reference in New Issue
Block a user