add --refresh to scanner

fdroidserver/scanner.py

@@ -21,7 +21,6 @@ import re
 import sys
 import json
 import imghdr
-import shutil
 import logging
 import zipfile
 import itertools
@@ -147,10 +146,12 @@ class SignatureDataVersionMismatchException(Exception):
 
 
 class SignatureDataController:
-    def __init__(self, name, filename):
+    def __init__(self, name, filename, url):
         self.name = name
         self.filename = filename
-        self.cache_outdated_interval = None
+        self.url = url
+        # by default we assume cache is valid indefinitely
+        self.cache_outdated_interval = timedelta(days=999999)
         self.data = {}
 
     def check_data_version(self):
@@ -165,7 +166,7 @@ class SignatureDataController:
         is not ok.
 
         :raises SignatureDataMalformedException: when timestamp value is
-                                                 inaccessible or not parseable
+                                                 inaccessible or not parse-able
         :raises SignatureDataOutdatedException: when timestamp is older then
                                                 `self.cache_outdated_interval`
         '''
@@ -178,9 +179,8 @@ class SignatureDataController:
             raise SignatureDataMalformedException() from e
         except TypeError as e:
             raise SignatureDataMalformedException() from e
-        if self.cache_outdated_interval:
+        if (timestamp + self.cache_outdated_interval) < scanner._datetime_now():
-            if (timestamp + self.cache_outdated_interval) < scanner._datetime_now():
+            raise SignatureDataOutdatedException()
-                raise SignatureDataOutdatedException()
 
     def fetch(self):
         try:
@@ -233,24 +233,31 @@ class SignatureDataController:
         self.check_data_version()
         valid_keys = ['timestamp', 'version', 'signatures']
 
-        for k in [x for x in self.data.keys() if x not in valid_keys]:
+        for k in list(self.data.keys()):
-            del self.data[k]
+            if k not in valid_keys:
+                del self.data[k]
+
+    def fetch_signatures_from_web(self):
+        logging.debug(_("downloading '{}'").format(self.url))
+        with urllib.request.urlopen(self.url) as f:
+            self.data = json.load(f)
 
 
 class ExodusSignatureDataController(SignatureDataController):
     def __init__(self):
-        super().__init__('Exodus signatures', 'exodus.yml')
+        super().__init__('Exodus signatures', 'exodus.yml', 'https://reports.exodus-privacy.eu.org/api/trackers')
         self.cache_outdated_interval = timedelta(days=1)  # refresh exodus cache after one day
 
     def fetch_signatures_from_web(self):
-        exodus_url = "https://reports.exodus-privacy.eu.org/api/trackers"
+        logging.debug(_("downloading '{}'").format(self.url))
+
         self.data = {
             "signatures": {},
             "timestamp": scanner._datetime_now().isoformat(),
             "version": SCANNER_CACHE_VERSION,
         }
 
-        with urllib.request.urlopen(exodus_url) as f:
+        with urllib.request.urlopen(self.url) as f:
             d = json.load(f)
             for tracker in d["trackers"].values():
                 if tracker.get('code_signature'):
@@ -261,21 +268,26 @@ class ExodusSignatureDataController(SignatureDataController):
                         # "network_signatures": [tracker["network_signature"]],
                         "AntiFeatures": ["Tracking"],
                         "license": "NonFree"  # We assume all trackers in exodus
-                                              # are non-free, alought free
+                                              # are non-free, although free
                                               # trackers like piwik, acra,
                                               # etc. might be listed by exodus
                                               # too.
                     }
 
 
-class SUSSDataController(SignatureDataController):
-    def __init__(self):
-        super().__init__('SUSS', 'suss.json')
-
-
 class ScannerTool():
     def __init__(self):
-        self.sdcs = [SUSSDataController()]
+        self.sdcs = [
+            SignatureDataController(
+                'SUSS',
+                'suss.json',
+                'https://fdroid.gitlab.io/fdroid-suss/suss.json'
+            ),
+        ]
+
+        # we could add support for loading additional signature source
+        # definitions from config.yml here
+
         self.load()
         self.compile_regexes()
 
@@ -301,11 +313,9 @@ class ScannerTool():
                 for sig in sigdef.get('warn_gradle_signatures', []):
                     self.regexs['warn_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE)
 
-    def clear_cache(self):
+    def refresh(self):
-        # delete cache folder and all its contents
+        for sdc in self.sdcs:
-        shutil.rmtree(scanner._scanner_cachedir(), ignore_errors=True)
+            sdc.fetch_signatures_from_web()
-        # re-initialize, this will re-populate the cache from default values
-        self.__init__()
 
     def add(self, new_controller: SignatureDataController):
         self.sdcs.append(new_controller)
@@ -660,8 +670,8 @@ def main():
                         help=_("Force scan of disabled apps and builds."))
     parser.add_argument("--json", action="store_true", default=False,
                         help=_("Output JSON to stdout."))
-    parser.add_argument("--clear-cache", action="store_true", default=False,
+    parser.add_argument("--refresh", "-r", action="store_true", default=False,
-                        help=_("purge local scanner definitions cache"))
+                        help=_("fetach the latest version of signatures from the web"))
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W
@@ -676,11 +686,14 @@ def main():
     # initialize/load configuration values
     common.get_config(options)
 
-    if options.clear_cache:
+    if options.refresh:
-        scanner._get_tool().clear_cache()
+        scanner._get_tool().refresh()
     if options.exodus:
         c = ExodusSignatureDataController()
-        c.fetch()
+        if options.refresh:
+            c.fetch_signatures_from_web()
+        else:
+            c.fetch()
         scanner._get_tool().add(c)
 
     probcount = 0

tests/scanner.TestCase

@@ -4,7 +4,6 @@ import glob
 import inspect
 import logging
 import optparse
-import io
 import os
 import re
 import shutil
@@ -500,56 +499,56 @@ class Test_scan_binary(unittest.TestCase):
 class Test_SignatureDataController(unittest.TestCase):
     # __init__
     def test_init(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         self.assertEqual(sdc.name, 'nnn')
         self.assertEqual(sdc.filename, 'fff.yml')
-        self.assertEqual(sdc.cache_outdated_interval, None)
+        self.assertEqual(sdc.cache_outdated_interval, timedelta(999999))
         self.assertDictEqual(sdc.data, {})
 
     # check_last_updated
     def test_check_last_updated_ok(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         sdc.data['timestamp'] = datetime.now().astimezone().isoformat()
         sdc.check_last_updated()
 
     def test_check_last_updated_exception_cache_outdated(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         sdc.cache_outdated_interval = timedelta(days=7)
         sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat()
         with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException):
             sdc.check_last_updated()
 
     def test_check_last_updated_exception_missing_timestamp_value(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
             sdc.check_last_updated()
 
     def test_check_last_updated_exception_not_string(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         sdc.data['timestamp'] = 12345
         with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
             sdc.check_last_updated()
 
     def test_check_last_updated_exception_not_iso_formatted_string(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         sdc.data['timestamp'] = '01/09/2002 10:11'
         with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
             sdc.check_last_updated()
 
     # check_data_version
     def test_check_data_version_ok(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         sdc.data['version'] = fdroidserver.scanner.SCANNER_CACHE_VERSION
         sdc.check_data_version()
 
     def test_check_data_version_exception(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         with self.assertRaises(fdroidserver.scanner.SignatureDataVersionMismatchException):
             sdc.check_data_version()
 
     def test_write_to_cache(self):
         open_func = mock.mock_open()
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml')
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         sdc.data = {"mocked": "data"}
 
         with mock.patch("builtins.open", open_func), mock.patch(
@@ -565,28 +564,6 @@ class Test_SignatureDataController(unittest.TestCase):
         )
 
 
-class Test_SUSSDataController_fetch_signatures_from_web(unittest.TestCase):
-    def setUp(self):
-        self.uo_func = mock.Mock(return_value=io.StringIO(textwrap.dedent('''\
-          version: 999
-          timestamp: "1999-12-31T23:59:59.999999+00:00"
-          signatures:
-          - binary_signature: com/google/firebase
-            name: Google Firebase
-            types:
-            - tracker
-            - non-free
-          - gradle_signature: com/google/android/gms
-            name: Google Mobile Services
-            types:
-            - non-free
-          - network_signature: doubleclick\\.net
-            name: Another thing to test.
-            types:
-            - ads
-        ''')))
-
-
 class Test_main(unittest.TestCase):
     def setUp(self):
         self.args = ["com.example.app", "local/additional.apk", "another.apk"]
@@ -654,7 +631,6 @@ if __name__ == "__main__":
         unittest.makeSuite(ScannerTest),
         unittest.makeSuite(Test_scan_binary),
         unittest.makeSuite(Test_SignatureDataController),
-        unittest.makeSuite(Test_SUSSDataController_fetch_signatures_from_web),
         unittest.makeSuite(Test_main),
     ])
     unittest.main(failfast=False)