mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-07-07 09:50:07 +02:00
507 lines
20 KiB
Python
Executable File
507 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import glob
|
|
import inspect
|
|
import logging
|
|
import optparse
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
import uuid
|
|
import yaml
|
|
import collections
|
|
import pathlib
|
|
from unittest import mock
|
|
|
|
localmodule = os.path.realpath(
|
|
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
|
)
|
|
print('localmodule: ' + localmodule)
|
|
if localmodule not in sys.path:
|
|
sys.path.insert(0, localmodule)
|
|
|
|
import fdroidserver.build
|
|
import fdroidserver.common
|
|
import fdroidserver.metadata
|
|
import fdroidserver.scanner
|
|
from testcommon import TmpCwd
|
|
|
|
|
|
class ScannerTest(unittest.TestCase):
|
|
def setUp(self):
|
|
logging.basicConfig(level=logging.INFO)
|
|
self.basedir = os.path.join(localmodule, 'tests')
|
|
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
|
|
if not os.path.exists(self.tmpdir):
|
|
os.makedirs(self.tmpdir)
|
|
os.chdir(self.basedir)
|
|
|
|
def test_scan_source_files(self):
|
|
fdroidserver.scanner.options = mock.Mock()
|
|
fdroidserver.scanner.options.json = False
|
|
source_files = os.path.join(self.basedir, 'source-files')
|
|
projects = {
|
|
'cn.wildfirechat.chat': 4,
|
|
'com.integreight.onesheeld': 11,
|
|
'Zillode': 1,
|
|
'firebase-suspect': 1,
|
|
'org.mozilla.rocket': 3,
|
|
'realm': 1,
|
|
'se.manyver': 2,
|
|
'com.jens.automation2': 2,
|
|
'com.github.shadowsocks': 6,
|
|
'org.tasks': 1,
|
|
'OtakuWorld': 2,
|
|
}
|
|
for d in glob.glob(os.path.join(source_files, '*')):
|
|
build = fdroidserver.metadata.Build()
|
|
fatal_problems = fdroidserver.scanner.scan_source(d, build)
|
|
should = projects.get(os.path.basename(d), 0)
|
|
self.assertEqual(
|
|
should, fatal_problems, "%s should have %d errors!" % (d, should)
|
|
)
|
|
|
|
def test_get_gradle_compile_commands(self):
|
|
test_files = [
|
|
('source-files/fdroid/fdroidclient/build.gradle', 'yes', 18),
|
|
('source-files/com.nextcloud.client/build.gradle', 'generic', 28),
|
|
('source-files/com.kunzisoft.testcase/build.gradle', 'libre', 4),
|
|
('source-files/cn.wildfirechat.chat/chat/build.gradle', 'yes', 33),
|
|
('source-files/org.tasks/app/build.gradle.kts', 'generic', 39),
|
|
('source-files/at.bitfire.davdroid/build.gradle', 'standard', 16),
|
|
('source-files/se.manyver/android/app/build.gradle', 'indie', 29),
|
|
('source-files/osmandapp/osmand/build.gradle', 'free', 5),
|
|
('source-files/eu.siacs.conversations/build.gradle', 'free', 24),
|
|
('source-files/org.mozilla.rocket/app/build.gradle', 'focus', 42),
|
|
('source-files/com.jens.automation2/app/build.gradle', 'fdroidFlavor', 8),
|
|
]
|
|
|
|
for f, flavor, count in test_files:
|
|
i = 0
|
|
build = fdroidserver.metadata.Build()
|
|
build.gradle = [flavor]
|
|
regexs = fdroidserver.scanner.get_gradle_compile_commands(build)
|
|
with open(f, encoding='utf-8') as fp:
|
|
for line in fp.readlines():
|
|
for regex in regexs:
|
|
m = regex.match(line)
|
|
if m:
|
|
i += 1
|
|
self.assertEqual(count, i)
|
|
|
|
def test_scan_source_files_sneaky_maven(self):
|
|
"""Check for sneaking in banned maven repos"""
|
|
testdir = tempfile.mkdtemp(
|
|
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
|
)
|
|
os.chdir(testdir)
|
|
fdroidserver.scanner.config = None
|
|
fdroidserver.scanner.options = mock.Mock()
|
|
fdroidserver.scanner.options.json = True
|
|
with open('build.gradle', 'w', encoding='utf-8') as fp:
|
|
fp.write(
|
|
textwrap.dedent(
|
|
"""
|
|
maven {
|
|
"https://jitpack.io"
|
|
url 'https://maven.fabric.io/public'
|
|
}
|
|
maven {
|
|
"https://maven.google.com"
|
|
setUrl('https://evilcorp.com/maven')
|
|
}
|
|
"""
|
|
)
|
|
)
|
|
count = fdroidserver.scanner.scan_source(testdir)
|
|
self.assertEqual(2, count, 'there should be this many errors')
|
|
|
|
def test_scan_source_file_types(self):
|
|
"""Build product files are not allowed, test they are detected
|
|
|
|
This test runs as if `fdroid build` running to test the
|
|
difference between absolute and relative paths.
|
|
|
|
"""
|
|
testdir = tempfile.mkdtemp(
|
|
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
|
)
|
|
build_dir = os.path.join('build', 'fake.app')
|
|
abs_build_dir = os.path.join(testdir, build_dir)
|
|
os.makedirs(abs_build_dir, exist_ok=True)
|
|
os.chdir(abs_build_dir)
|
|
|
|
fdroidserver.scanner.config = None
|
|
fdroidserver.scanner.options = mock.Mock()
|
|
fdroidserver.scanner.options.json = True
|
|
|
|
keep = [
|
|
'arg.jar',
|
|
'ascii.out',
|
|
'baz.so',
|
|
'classes.dex',
|
|
'sqlcipher.aar',
|
|
'static.a',
|
|
'src/test/resources/classes.dex',
|
|
]
|
|
remove = ['gradle-wrapper.jar', 'gradlew', 'gradlew.bat']
|
|
os.makedirs('src/test/resources', exist_ok=True)
|
|
for f in keep + remove:
|
|
with open(f, 'w') as fp:
|
|
fp.write('placeholder')
|
|
self.assertTrue(os.path.exists(f))
|
|
binaries = ['binary.out', 'fake.png', 'snippet.png']
|
|
with open('binary.out', 'wb') as fp:
|
|
fp.write(b'\x00\x00')
|
|
fp.write(uuid.uuid4().bytes)
|
|
shutil.copyfile('binary.out', 'fake.png')
|
|
os.chmod('fake.png', 0o755)
|
|
os.system('ls -l binary.out')
|
|
with open('snippet.png', 'wb') as fp:
|
|
fp.write(
|
|
b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x000\x00\x00'
|
|
b'\x000\x08\x06\x00\x00\x00W\x02\xf9\x87\x00\x00\x00\x04sB'
|
|
b'IT\x08\x08\x08\x08|\x08d\x88\x00\x00\x00\tpHYs\x00\x00\n'
|
|
b'a\x00\x00\na\x01\xfc\xccJ%\x00\x00\x00\x19tEXtSoftware'
|
|
)
|
|
os.chmod('snippet.png', 0o755)
|
|
os.system('ls -l fake.png')
|
|
|
|
# run scanner as if from `fdroid build`
|
|
os.chdir(testdir)
|
|
count = fdroidserver.scanner.scan_source(build_dir)
|
|
self.assertEqual(6, count, 'there should be this many errors')
|
|
os.chdir(build_dir)
|
|
|
|
for f in keep + binaries:
|
|
self.assertTrue(os.path.exists(f), f + ' should still be there')
|
|
for f in remove:
|
|
self.assertFalse(os.path.exists(f), f + ' should have been removed')
|
|
|
|
files = dict()
|
|
for section in ('errors', 'infos', 'warnings'):
|
|
files[section] = []
|
|
for msg, f in fdroidserver.scanner.json_per_build[section]:
|
|
files[section].append(f)
|
|
|
|
self.assertFalse('ascii.out' in files['errors'],
|
|
'an ASCII .out file is not an error')
|
|
self.assertFalse('snippet.png' in files['errors'],
|
|
'an executable valid image is not an error')
|
|
|
|
self.assertTrue('arg.jar' in files['errors'], 'all JAR files are errors')
|
|
self.assertTrue('baz.so' in files['errors'], 'all .so files are errors')
|
|
self.assertTrue('binary.out' in files['errors'], 'a binary .out file is an error')
|
|
self.assertTrue('classes.dex' in files['errors'], 'all classes.dex files are errors')
|
|
self.assertTrue('sqlcipher.aar' in files['errors'], 'all AAR files are errors')
|
|
self.assertTrue('static.a' in files['errors'], 'all .a files are errors')
|
|
|
|
self.assertTrue('fake.png' in files['warnings'],
|
|
'a random binary that is executable that is not an image is a warning')
|
|
self.assertTrue('src/test/resources/classes.dex' in files['warnings'],
|
|
'suspicious file but in a test dir is a warning')
|
|
|
|
for f in remove:
|
|
self.assertTrue(f in files['infos'],
|
|
f + ' should be removed with an info message')
|
|
|
|
def test_build_local_scanner(self):
|
|
"""`fdroid build` calls scanner functions, test them here"""
|
|
testdir = tempfile.mkdtemp(
|
|
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
|
)
|
|
os.chdir(testdir)
|
|
|
|
config = dict()
|
|
fdroidserver.common.fill_config_defaults(config)
|
|
fdroidserver.common.config = config
|
|
fdroidserver.build.config = config
|
|
fdroidserver.build.options = mock.Mock()
|
|
fdroidserver.build.options.json = False
|
|
fdroidserver.build.options.scan_binary = False
|
|
fdroidserver.build.options.notarball = True
|
|
fdroidserver.build.options.skipscan = False
|
|
fdroidserver.scanner.options = fdroidserver.build.options
|
|
|
|
app = fdroidserver.metadata.App()
|
|
app.id = 'mocked.app.id'
|
|
build = fdroidserver.metadata.Build()
|
|
build.commit = '1.0'
|
|
build.output = app.id + '.apk'
|
|
build.scanignore = ['baz.so', 'foo.aar']
|
|
build.versionCode = '1'
|
|
build.versionName = '1.0'
|
|
vcs = mock.Mock()
|
|
|
|
for f in ('baz.so', 'foo.aar', 'gradle-wrapper.jar'):
|
|
with open(f, 'w') as fp:
|
|
fp.write('placeholder')
|
|
self.assertTrue(os.path.exists(f))
|
|
|
|
with open('build.xml', 'w', encoding='utf-8') as fp:
|
|
fp.write(
|
|
textwrap.dedent(
|
|
"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
<project basedir="." default="clean" name="mockapp">
|
|
<target name="release"/>
|
|
<target name="clean"/>
|
|
</project>"""
|
|
)
|
|
)
|
|
|
|
def make_fake_apk(output, build):
|
|
with open(build.output, 'w') as fp:
|
|
fp.write('APK PLACEHOLDER')
|
|
return output
|
|
|
|
with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk):
|
|
with mock.patch('fdroidserver.common.get_native_code', return_value='x86'):
|
|
with mock.patch('fdroidserver.common.get_apk_id',
|
|
return_value=(app.id, build.versionCode, build.versionName)):
|
|
with mock.patch('fdroidserver.common.is_apk_and_debuggable', return_value=False):
|
|
fdroidserver.build.build_local(
|
|
app, build, vcs,
|
|
build_dir=testdir, output_dir=testdir,
|
|
log_dir=None, srclib_dir=None, extlib_dir=None, tmp_dir=None,
|
|
force=False, onserver=False, refresh=False
|
|
)
|
|
self.assertTrue(os.path.exists('baz.so'))
|
|
self.assertTrue(os.path.exists('foo.aar'))
|
|
self.assertFalse(os.path.exists('gradle-wrapper.jar'))
|
|
|
|
def test_gradle_maven_url_regex(self):
|
|
"""Check the regex can find all the cases"""
|
|
with open(os.path.join(self.basedir, 'gradle-maven-blocks.yaml')) as fp:
|
|
data = yaml.safe_load(fp)
|
|
|
|
urls = []
|
|
for entry in data:
|
|
found = False
|
|
for m in fdroidserver.scanner.MAVEN_URL_REGEX.findall(entry):
|
|
urls.append(m)
|
|
found = True
|
|
self.assertTrue(found, 'this block should produce a URL:\n' + entry)
|
|
self.assertEqual(len(data), len(urls), 'each data example should produce a URL')
|
|
|
|
def test_scan_gradle_file_with_multiple_problems(self):
|
|
"""Check that the scanner can handle scandelete with gradle files with multiple problems"""
|
|
testdir = tempfile.mkdtemp(
|
|
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
|
)
|
|
os.chdir(testdir)
|
|
fdroidserver.scanner.config = None
|
|
fdroidserver.scanner.options = mock.Mock()
|
|
build = fdroidserver.metadata.Build()
|
|
build.scandelete = ['build.gradle']
|
|
with open('build.gradle', 'w', encoding='utf-8') as fp:
|
|
fp.write(
|
|
textwrap.dedent(
|
|
"""
|
|
maven {
|
|
url 'https://maven.fabric.io/public'
|
|
}
|
|
maven {
|
|
url 'https://evilcorp.com/maven'
|
|
}
|
|
"""
|
|
)
|
|
)
|
|
count = fdroidserver.scanner.scan_source(testdir, build)
|
|
self.assertFalse(os.path.exists("build.gradle"))
|
|
self.assertEqual(0, count, 'there should be this many errors')
|
|
|
|
|
|
class Test_scan_binary(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.basedir = os.path.join(localmodule, 'tests')
|
|
config = dict()
|
|
fdroidserver.common.fill_config_defaults(config)
|
|
fdroidserver.common.config = config
|
|
fdroidserver.common.options = mock.Mock()
|
|
|
|
def test_code_signature_match(self):
|
|
apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk')
|
|
with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", {"java/lang/Object": re.compile(
|
|
r'.*java/lang/Object', re.IGNORECASE | re.UNICODE
|
|
)}):
|
|
self.assertEqual(
|
|
1,
|
|
fdroidserver.scanner.scan_binary(apkfile),
|
|
"Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile),
|
|
)
|
|
|
|
@unittest.skipIf(
|
|
sys.version_info < (3, 9),
|
|
"Our implementation for traversing zip files will silently fail to work"
|
|
"on older python versions, also see: "
|
|
"https://gitlab.com/fdroid/fdroidserver/-/merge_requests/1110#note_932026766"
|
|
)
|
|
def test_bottom_level_embedded_apk_code_signature(self):
|
|
apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk')
|
|
with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", {"org/bitbucket/tickytacky/mirrormirror/MainActivity": re.compile(
|
|
r'.*org/bitbucket/tickytacky/mirrormirror/MainActivity', re.IGNORECASE | re.UNICODE
|
|
)}):
|
|
self.assertEqual(
|
|
1,
|
|
fdroidserver.scanner.scan_binary(apkfile),
|
|
"Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile),
|
|
)
|
|
|
|
def test_top_level_signature_embedded_apk_present(self):
|
|
apkfile = os.path.join(self.basedir, 'apk.embedded_1.apk')
|
|
with mock.patch("fdroidserver.scanner.CODE_SIGNATURES", {"org/fdroid/ci/BuildConfig": re.compile(
|
|
r'.*org/fdroid/ci/BuildConfig', re.IGNORECASE | re.UNICODE
|
|
)}):
|
|
self.assertEqual(
|
|
1,
|
|
fdroidserver.scanner.scan_binary(apkfile),
|
|
"Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner.CODE_SIGNATURES.values(), apkfile),
|
|
)
|
|
|
|
def test_no_match(self):
|
|
apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk')
|
|
result = fdroidserver.scanner.scan_binary(apkfile)
|
|
self.assertEqual(0, result, "Found false positives in binary '{}'".format(apkfile))
|
|
|
|
|
|
class Test__exodus_compile_signatures(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.m1 = mock.Mock()
|
|
self.m1.code_signature = r"^random\sregex$"
|
|
self.m2 = mock.Mock()
|
|
self.m2.code_signature = r"^another.+regex$"
|
|
self.mock_sigs = [self.m1, self.m2]
|
|
|
|
def test_ok(self):
|
|
result = fdroidserver.scanner._exodus_compile_signatures(self.mock_sigs)
|
|
self.assertListEqual(result, [
|
|
re.compile(self.m1.code_signature),
|
|
re.compile(self.m2.code_signature),
|
|
])
|
|
|
|
def test_not_iterable(self):
|
|
result = fdroidserver.scanner._exodus_compile_signatures(123)
|
|
self.assertListEqual(result, [])
|
|
|
|
|
|
class Test_load_exodus_trackers_signatures(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.requests_ret = mock.Mock()
|
|
self.requests_ret.json = mock.Mock(return_value={
|
|
"trackers": {
|
|
"1": {
|
|
"id": 1,
|
|
"name": "Steyer Puch 1",
|
|
"description": "blah blah blah",
|
|
"creation_date": "1956-01-01",
|
|
"code_signature": "com.puch.|com.steyer.",
|
|
"network_signature": "pst\\.com",
|
|
"website": "https://pst.com",
|
|
"categories": ["tracker"],
|
|
"documentation": [],
|
|
},
|
|
"2": {
|
|
"id": 2,
|
|
"name": "Steyer Puch 2",
|
|
"description": "blah blah blah",
|
|
"creation_date": "1956-01-01",
|
|
"code_signature": "com.puch.|com.steyer.",
|
|
"network_signature": "pst\\.com",
|
|
"website": "https://pst.com",
|
|
"categories": ["tracker"],
|
|
"documentation": [],
|
|
}
|
|
},
|
|
})
|
|
self.requests_func = mock.Mock(return_value=self.requests_ret)
|
|
self.compilesig_func = mock.Mock(return_value="mocked return value")
|
|
|
|
def test_ok(self):
|
|
with mock.patch("requests.get", self.requests_func), mock.patch(
|
|
"fdroidserver.scanner._exodus_compile_signatures", self.compilesig_func
|
|
):
|
|
result_sigs, result_regex = fdroidserver.scanner.load_exodus_trackers_signatures()
|
|
self.requests_func.assert_called_once_with("https://reports.exodus-privacy.eu.org/api/trackers")
|
|
self.assertEqual(len(result_sigs), 2)
|
|
self.assertListEqual([1, 2], sorted([x.id for x in result_sigs]))
|
|
|
|
self.compilesig_func.assert_called_once_with(result_sigs)
|
|
self.assertEqual(result_regex, "mocked return value")
|
|
|
|
|
|
class Test_main(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.args = ["com.example.app", "local/additional.apk", "another.apk"]
|
|
self.exit_func = mock.Mock()
|
|
self.read_app_args_func = mock.Mock(return_value={})
|
|
self.scan_binary_func = mock.Mock(return_value=0)
|
|
|
|
def test_parsing_appid(self):
|
|
"""
|
|
This test verifies that app id get parsed correctly
|
|
(doesn't test how they get processed)
|
|
"""
|
|
self.args = ["com.example.app"]
|
|
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir), mock.patch(
|
|
"sys.exit", self.exit_func
|
|
), mock.patch("sys.argv", ["fdroid scanner", *self.args]), mock.patch(
|
|
"fdroidserver.common.read_app_args", self.read_app_args_func
|
|
), mock.patch("fdroidserver.scanner.scan_binary", self.scan_binary_func):
|
|
fdroidserver.scanner.main()
|
|
|
|
self.exit_func.assert_not_called()
|
|
self.read_app_args_func.assert_called_once_with(
|
|
['com.example.app'], collections.OrderedDict(), True
|
|
)
|
|
self.scan_binary_func.assert_not_called()
|
|
|
|
def test_parsing_apkpath(self):
|
|
"""
|
|
This test verifies that apk paths get parsed correctly
|
|
(doesn't test how they get processed)
|
|
"""
|
|
self.args = ["local.application.apk"]
|
|
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir), mock.patch(
|
|
"sys.exit", self.exit_func
|
|
), mock.patch("sys.argv", ["fdroid scanner", *self.args]), mock.patch(
|
|
"fdroidserver.common.read_app_args", self.read_app_args_func
|
|
), mock.patch("fdroidserver.scanner.scan_binary", self.scan_binary_func):
|
|
pathlib.Path(self.args[0]).touch()
|
|
fdroidserver.scanner.main()
|
|
|
|
self.exit_func.assert_not_called()
|
|
self.read_app_args_func.assert_not_called()
|
|
self.scan_binary_func.assert_called_once_with('local.application.apk', [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
os.chdir(os.path.dirname(__file__))
|
|
|
|
parser = optparse.OptionParser()
|
|
parser.add_option(
|
|
"-v",
|
|
"--verbose",
|
|
action="store_true",
|
|
default=False,
|
|
help="Spew out even more information than normal",
|
|
)
|
|
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
|
|
|
|
newSuite = unittest.TestSuite()
|
|
newSuite.addTests([
|
|
unittest.makeSuite(ScannerTest),
|
|
unittest.makeSuite(Test_scan_binary),
|
|
unittest.makeSuite(Test__exodus_compile_signatures),
|
|
unittest.makeSuite(Test_load_exodus_trackers_signatures),
|
|
unittest.makeSuite(Test_main),
|
|
])
|
|
unittest.main(failfast=False)
|