scanner: implement caching rules for suss

fdroidserver/build.py

@@ -987,7 +987,7 @@ def main():
         if not options.appid and not options.all:
             parser.error("option %s: If you really want to build all the apps, use --all" % "all")
 
-    config = common.read_config(options)
+    config = common.read_config(opts=options)
 
     if config['build_server_always']:
         options.server = True

fdroidserver/common.py

@@ -324,19 +324,26 @@ def fill_config_defaults(thisconfig):
         thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle')
 
 
-def get_config(options=None):
+def get_config(opts=None):
     """
     helper function for getting access to commons.config while safely
     initializing if it wasn't initialized yet.
     """
-    global config
+    global config, options
 
     if config is not None:
         return config
 
     config = {}
     common.fill_config_defaults(config)
-    common.read_config(options)
+    common.read_config(opts=opts)
+
+    # make sure these values are available in common.py even if they didn't
+    # declare global in a scope
+    common.config = config
+    if opts is not None:
+        common.options = opts
+
     return config
 
 

fdroidserver/scanner.py

@@ -141,6 +141,10 @@ class SignatureDataOutdatedException(Exception):
     pass
 
 
+class SignatureDataCacheMissException(Exception):
+    pass
+
+
 class SignatureDataVersionMismatchException(Exception):
     pass
 
@@ -151,7 +155,7 @@ class SignatureDataController:
         self.filename = filename
         self.url = url
         # by default we assume cache is valid indefinitely
-        self.cache_outdated_interval = timedelta(days=999999)
+        self.cache_duration = timedelta(days=999999)
         self.data = {}
 
     def check_data_version(self):
@@ -162,63 +166,65 @@ class SignatureDataController:
         '''
         NOTE: currently not in use
 
-        Checks if the timestamp value is ok. Raises an exception if something
+        Checks if the last_updated value is ok. Raises an exception if
-        is not ok.
+        it's expired or inaccessible.
 
         :raises SignatureDataMalformedException: when timestamp value is
                                                  inaccessible or not parse-able
         :raises SignatureDataOutdatedException: when timestamp is older then
-                                                `self.cache_outdated_interval`
+                                                `self.cache_duration`
         '''
-        timestamp = self.data.get("timestamp")
+        last_updated = self.data.get("last_updated", None)
-        if not timestamp:
+        if last_updated:
-            raise SignatureDataMalformedException()
+            try:
-        try:
+                last_updated = datetime.fromisoformat(last_updated)
-            timestamp = datetime.fromisoformat(timestamp)
+            except ValueError as e:
-        except ValueError as e:
+                raise SignatureDataMalformedException() from e
-            raise SignatureDataMalformedException() from e
+            except TypeError as e:
-        except TypeError as e:
+                raise SignatureDataMalformedException() from e
-            raise SignatureDataMalformedException() from e
+            delta = (last_updated + self.cache_duration) - scanner._datetime_now()
-        if (timestamp + self.cache_outdated_interval) < scanner._datetime_now():
+            if delta > timedelta(seconds=0):
-            raise SignatureDataOutdatedException()
+                logging.debug(_('next {name} cache update due in {time}').format(
+                    name=self.filename, time=delta
+                ))
+            else:
+                raise SignatureDataOutdatedException()
 
     def fetch(self):
         try:
-            self.load_from_cache()
+            self.fetch_signatures_from_web()
-            self.verify_data()
-            self.check_last_updated()
-        except (
-            SignatureDataMalformedException,
-            SignatureDataVersionMismatchException,
-            SignatureDataOutdatedException
-        ):
-            try:
-                self.fetch_signatures_from_web()
-            except AttributeError:
-                # just load from defaults if fetch_signatures_from_web is not
-                # implemented
-                self.load_from_defaults()
             self.write_to_cache()
+        except Exception as e:
+            raise Exception(_("downloading scanner signatures from '{}' failed").format(self.url)) from e
 
     def load(self):
         try:
-            self.load_from_cache()
+            try:
-            self.verify_data()
+                self.load_from_cache()
-        except (SignatureDataMalformedException, SignatureDataVersionMismatchException):
+                self.verify_data()
-            self.load_from_defaults()
+                self.check_last_updated()
+            except SignatureDataCacheMissException:
+                self.load_from_defaults()
+        except SignatureDataOutdatedException:
+            self.fetch_signatures_from_web()
             self.write_to_cache()
+        except (SignatureDataMalformedException, SignatureDataVersionMismatchException) as e:
+            logging.critical(_("scanner cache is malformed! You can clear it with: '{clear}'").format(
+                clear='rm -r {}'.format(common.get_config()['cachedir_scanner'])
+            ))
+            raise e
 
     def load_from_defaults(self):
         sig_file = (Path(__file__).parent / 'data' / 'scanner' / self.filename).resolve()
         with open(sig_file) as f:
-            self.data = json.load(f)
+            self.set_data(json.load(f))
 
     def load_from_cache(self):
         sig_file = scanner._scanner_cachedir() / self.filename
         if not sig_file.exists():
-            raise SignatureDataMalformedException()
+            raise SignatureDataCacheMissException()
         with open(sig_file) as f:
-            self.data = json.load(f)
+            self.set_data(json.load(f))
 
     def write_to_cache(self):
         sig_file = scanner._scanner_cachedir() / self.filename
@@ -231,29 +237,36 @@ class SignatureDataController:
         cleans and validates and cleans `self.data`
         '''
         self.check_data_version()
-        valid_keys = ['timestamp', 'version', 'signatures']
+        valid_keys = ['timestamp', 'last_updated', 'version', 'signatures', 'cache_duration']
 
         for k in list(self.data.keys()):
             if k not in valid_keys:
                 del self.data[k]
 
+    def set_data(self, new_data):
+        self.data = new_data
+        if 'cache_duration' in new_data:
+            self.cache_duration = timedelta(seconds=new_data['cache_duration'])
+
     def fetch_signatures_from_web(self):
         logging.debug(_("downloading '{}'").format(self.url))
         with urllib.request.urlopen(self.url) as f:
-            self.data = json.load(f)
+            self.set_data(json.load(f))
+        self.data['last_updated'] = scanner._datetime_now().isoformat()
 
 
 class ExodusSignatureDataController(SignatureDataController):
     def __init__(self):
         super().__init__('Exodus signatures', 'exodus.yml', 'https://reports.exodus-privacy.eu.org/api/trackers')
-        self.cache_outdated_interval = timedelta(days=1)  # refresh exodus cache after one day
+        self.cache_duration = timedelta(days=1)  # refresh exodus cache after one day
 
     def fetch_signatures_from_web(self):
         logging.debug(_("downloading '{}'").format(self.url))
 
-        self.data = {
+        data = {
             "signatures": {},
             "timestamp": scanner._datetime_now().isoformat(),
+            "last_updated": scanner._datetime_now().isoformat(),
             "version": SCANNER_CACHE_VERSION,
         }
 
@@ -261,7 +274,7 @@ class ExodusSignatureDataController(SignatureDataController):
             d = json.load(f)
             for tracker in d["trackers"].values():
                 if tracker.get('code_signature'):
-                    self.data["signatures"][tracker["name"]] = {
+                    data["signatures"][tracker["name"]] = {
                         "name": tracker["name"],
                         "warn_code_signatures": [tracker["code_signature"]],
                         # exodus also provides network signatures, unused atm.
@@ -273,6 +286,7 @@ class ExodusSignatureDataController(SignatureDataController):
                                               # etc. might be listed by exodus
                                               # too.
                     }
+        self.set_data(data)
 
 
 class ScannerTool():
@@ -316,6 +330,7 @@ class ScannerTool():
     def refresh(self):
         for sdc in self.sdcs:
             sdc.fetch_signatures_from_web()
+            sdc.write_to_cache()
 
     def add(self, new_controller: SignatureDataController):
         self.sdcs.append(new_controller)
@@ -684,7 +699,7 @@ def main():
             logging.getLogger().setLevel(logging.ERROR)
 
     # initialize/load configuration values
-    common.get_config(options)
+    common.get_config(opts=options)
 
     if options.refresh:
         scanner._get_tool().refresh()

tests/scanner.TestCase

@@ -502,39 +502,38 @@ class Test_SignatureDataController(unittest.TestCase):
         sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
         self.assertEqual(sdc.name, 'nnn')
         self.assertEqual(sdc.filename, 'fff.yml')
-        self.assertEqual(sdc.cache_outdated_interval, timedelta(999999))
+        self.assertEqual(sdc.cache_duration, timedelta(999999))
         self.assertDictEqual(sdc.data, {})
 
     # check_last_updated
     def test_check_last_updated_ok(self):
         sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
-        sdc.data['timestamp'] = datetime.now().astimezone().isoformat()
+        sdc.data['last_updated'] = datetime.now().astimezone().isoformat()
         sdc.check_last_updated()
 
     def test_check_last_updated_exception_cache_outdated(self):
         sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
-        sdc.cache_outdated_interval = timedelta(days=7)
+        sdc.cache_duration = timedelta(days=7)
-        sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat()
+        sdc.data['last_updated'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat()
         with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException):
             sdc.check_last_updated()
 
-    def test_check_last_updated_exception_missing_timestamp_value(self):
-        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
-        with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
-            sdc.check_last_updated()
-
     def test_check_last_updated_exception_not_string(self):
         sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
-        sdc.data['timestamp'] = 12345
+        sdc.data['last_updated'] = 12345
         with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
             sdc.check_last_updated()
 
     def test_check_last_updated_exception_not_iso_formatted_string(self):
         sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
-        sdc.data['timestamp'] = '01/09/2002 10:11'
+        sdc.data['last_updated'] = '01/09/2002 10:11'
         with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
             sdc.check_last_updated()
 
+    def test_check_last_updated_no_exception_missing_when_last_updated_not_set(self):
+        sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
+        sdc.check_last_updated()
+
     # check_data_version
     def test_check_data_version_ok(self):
         sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')