From c9b59b525d486662d6e007711a3814ef35b2f0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 28 Sep 2022 18:18:19 +0200 Subject: [PATCH] fix timestamp check; remove dead code --- fdroidserver/scanner.py | 40 ------------ tests/scanner.TestCase | 131 +--------------------------------------- 2 files changed, 2 insertions(+), 169 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 31344288..b4d46a4d 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -24,12 +24,10 @@ import imghdr import shutil import logging import zipfile -import requests import itertools import traceback import urllib.request from argparse import ArgumentParser -from collections import namedtuple from copy import deepcopy from tempfile import TemporaryDirectory from pathlib import Path @@ -115,23 +113,6 @@ def get_embedded_classes(apkfile, depth=0): return classes -# taken from exodus_core -def _exodus_compile_signatures(signatures): - """ - Compiles the regex associated to each signature, in order to speed up the trackers detection. - - :return: A compiled list of signatures. - """ - compiled_tracker_signature = [] - try: - compiled_tracker_signature = [ - re.compile(track.code_signature) for track in signatures - ] - except TypeError: - print("signatures is not iterable") - return compiled_tracker_signature - - def _datetime_now(): """ simple wrapper for datetime.now to allow mocking it for testing @@ -347,27 +328,6 @@ def _get_tool(): return scanner._SCANNER_TOOL -# taken from exodus_core -def load_exodus_trackers_signatures(): - """ - Load trackers signatures from the official Exodus database. - - :return: a dictionary containing signatures. - """ - signatures = [] - exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers" - r = requests.get(exodus_url, timeout=300) - data = r.json() - for e in data['trackers']: - signatures.append( - namedtuple('tracker', data['trackers'][e].keys())( - *data['trackers'][e].values() - ) - ) - logging.debug('{} trackers signatures loaded'.format(len(signatures))) - return signatures, scanner._exodus_compile_signatures(signatures) - - def scan_binary(apkfile, extract_signatures=None): """Scan output of dexdump for known non-free classes.""" logging.info(_('Scanning APK with dexdump for known non-free classes.')) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index ade8bd0a..4922fdf3 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -496,132 +496,6 @@ class Test_scan_binary(unittest.TestCase): "Did not find expected code signature '{}' in binary '{}'".format(fdroidserver.scanner._SCANNER_TOOL.regexs['err_code_signatures'].values(), apkfile), ) - # TODO: re-enable once allow-listing migrated to more complex regexes - # def test_no_match(self): - # apkfile = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk') - # result = fdroidserver.scanner.scan_binary(apkfile) - # self.assertEqual(0, result, "Found false positives in binary '{}'".format(apkfile)) - - -# class Test__fetch_exodus_signatures_to_cache(unittest.TestCase): -# def setUp(self): -# self.web_req_func = mock.Mock(return_value=io.StringIO(json.dumps({ -# "trackers": { -# "1": { -# "id": 1, -# "name": "Steyer Puch 1", -# "description": "blah blah blah", -# "creation_date": "1956-01-01", -# "code_signature": "com.puch.|com.steyer.", -# "network_signature": "pst\\.com", -# "website": "https://pst.com", -# "categories": ["tracker"], -# "documentation": [], -# }, -# "2": { -# "id": 2, -# "name": "Steyer Puch 2", -# "description": "blah blah blah", -# "creation_date": "1956-01-01", -# "code_signature": "com.puch.|com.steyer.", -# "network_signature": "pst\\.com", -# "website": "https://pst.com", -# "categories": ["tracker"], -# "documentation": [], -# } -# }, -# }))) -# self.open_func = mock.mock_open() -# self.cachedir_func = mock.Mock(return_value=pathlib.Path("mocked/path")) -# -# def test_ok(self): -# with mock.patch("urllib.request.urlopen", self.web_req_func), mock.patch( -# "builtins.open", self.open_func -# ) as outfilemock, mock.patch( -# "fdroidserver.scanner._scanner_cachedir", self.cachedir_func -# ), mock.patch("fdroidserver.scanner._datetime_now", unittest.mock.Mock(return_value=datetime(1999, 12, 31, 23, 59, 59))): -# fdroidserver.scanner.fetch_exodus_signatures_to_cache() -# -# self.cachedir_func.assert_called_once() -# self.web_req_func.assert_called_once_with("https://reports.exodus-privacy.eu.org/api/trackers") -# self.open_func.assert_called_once_with(pathlib.Path("mocked/path/exodus.json"), "w", encoding="utf-8") -# self.assertEqual( -# mock_open_to_str(self.open_func), -# """{"signatures": {"exodus-1": {"name": "Steyer Puch 1", "code_signature": "com.puch.|com.steyer.", "network_signature": "pst\\\\.com", "types": ["tracker", "non-free"]}, "exodus-2": {"name": "Steyer Puch 2", "code_signature": "com.puch.|com.steyer.", "network_signature": "pst\\\\.com", "types": ["tracker", "non-free"]}}, "timestamp": "1999-12-31T23:59:59"}""" -# ) - - -class Test__exodus_compile_signatures(unittest.TestCase): - def setUp(self): - self.m1 = mock.Mock() - self.m1.code_signature = r"^random\sregex$" - self.m2 = mock.Mock() - self.m2.code_signature = r"^another.+regex$" - self.mock_sigs = [self.m1, self.m2] - - def test_ok(self): - result = fdroidserver.scanner._exodus_compile_signatures(self.mock_sigs) - self.assertListEqual( - result, - [ - re.compile(self.m1.code_signature), - re.compile(self.m2.code_signature), - ], - ) - - def test_not_iterable(self): - result = fdroidserver.scanner._exodus_compile_signatures(123) - self.assertListEqual(result, []) - - -class Test_load_exodus_trackers_signatures(unittest.TestCase): - def setUp(self): - self.requests_ret = mock.Mock() - self.requests_ret.json = mock.Mock( - return_value={ - "trackers": { - "1": { - "id": 1, - "name": "Steyer Puch 1", - "description": "blah blah blah", - "creation_date": "1956-01-01", - "code_signature": "com.puch.|com.steyer.", - "network_signature": "pst\\.com", - "website": "https://pst.com", - "categories": ["tracker"], - "documentation": [], - }, - "2": { - "id": 2, - "name": "Steyer Puch 2", - "description": "blah blah blah", - "creation_date": "1956-01-01", - "code_signature": "com.puch.|com.steyer.", - "network_signature": "pst\\.com", - "website": "https://pst.com", - "categories": ["tracker"], - "documentation": [], - }, - }, - } - ) - self.requests_func = mock.Mock(return_value=self.requests_ret) - self.compilesig_func = mock.Mock(return_value="mocked return value") - - def test_ok(self): - with mock.patch("requests.get", self.requests_func), mock.patch( - "fdroidserver.scanner._exodus_compile_signatures", self.compilesig_func - ): - sigs, regex = fdroidserver.scanner.load_exodus_trackers_signatures() - self.requests_func.assert_called_once_with( - "https://reports.exodus-privacy.eu.org/api/trackers", timeout=300 - ) - self.assertEqual(len(sigs), 2) - self.assertListEqual([1, 2], sorted([x.id for x in sigs])) - - self.compilesig_func.assert_called_once_with(sigs) - self.assertEqual(regex, "mocked return value") - class Test_SignatureDataController(unittest.TestCase): # __init__ @@ -629,7 +503,7 @@ class Test_SignatureDataController(unittest.TestCase): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') self.assertEqual(sdc.name, 'nnn') self.assertEqual(sdc.filename, 'fff.yml') - self.assertEqual(sdc.cache_outdated_interval, timedelta(days=7)) + self.assertEqual(sdc.cache_outdated_interval, None) self.assertDictEqual(sdc.data, {}) # check_last_updated @@ -640,6 +514,7 @@ class Test_SignatureDataController(unittest.TestCase): def test_check_last_updated_exception_cache_outdated(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml') + sdc.cache_outdated_interval = timedelta(days=7) sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() @@ -778,8 +653,6 @@ if __name__ == "__main__": newSuite.addTests([ unittest.makeSuite(ScannerTest), unittest.makeSuite(Test_scan_binary), - unittest.makeSuite(Test__exodus_compile_signatures), - unittest.makeSuite(Test_load_exodus_trackers_signatures), unittest.makeSuite(Test_SignatureDataController), unittest.makeSuite(Test_ScannerSignatureDataController_fetch_signatures_from_web), unittest.makeSuite(Test_main),