diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 32b3306f..ca37a4bb 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -987,7 +987,7 @@ def main(): if not options.appid and not options.all: parser.error("option %s: If you really want to build all the apps, use --all" % "all") - config = common.read_config(options) + config = common.read_config(opts=options) if config['build_server_always']: options.server = True diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7e6bed29..7793c51f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -324,19 +324,26 @@ def fill_config_defaults(thisconfig): thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle') -def get_config(options=None): +def get_config(opts=None): """ helper function for getting access to commons.config while safely initializing if it wasn't initialized yet. """ - global config + global config, options if config is not None: return config config = {} common.fill_config_defaults(config) - common.read_config(options) + common.read_config(opts=opts) + + # make sure these values are available in common.py even if they didn't + # declare global in a scope + common.config = config + if opts is not None: + common.options = opts + return config diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 9fc5fcb2..e0852eea 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -141,6 +141,10 @@ class SignatureDataOutdatedException(Exception): pass +class SignatureDataCacheMissException(Exception): + pass + + class SignatureDataVersionMismatchException(Exception): pass @@ -151,7 +155,7 @@ class SignatureDataController: self.filename = filename self.url = url # by default we assume cache is valid indefinitely - self.cache_outdated_interval = timedelta(days=999999) + self.cache_duration = timedelta(days=999999) self.data = {} def check_data_version(self): @@ -162,63 +166,65 @@ class SignatureDataController: ''' NOTE: currently not in use - Checks if the timestamp value is ok. Raises an exception if something - is not ok. + Checks if the last_updated value is ok. Raises an exception if + it's expired or inaccessible. :raises SignatureDataMalformedException: when timestamp value is inaccessible or not parse-able :raises SignatureDataOutdatedException: when timestamp is older then - `self.cache_outdated_interval` + `self.cache_duration` ''' - timestamp = self.data.get("timestamp") - if not timestamp: - raise SignatureDataMalformedException() - try: - timestamp = datetime.fromisoformat(timestamp) - except ValueError as e: - raise SignatureDataMalformedException() from e - except TypeError as e: - raise SignatureDataMalformedException() from e - if (timestamp + self.cache_outdated_interval) < scanner._datetime_now(): - raise SignatureDataOutdatedException() + last_updated = self.data.get("last_updated", None) + if last_updated: + try: + last_updated = datetime.fromisoformat(last_updated) + except ValueError as e: + raise SignatureDataMalformedException() from e + except TypeError as e: + raise SignatureDataMalformedException() from e + delta = (last_updated + self.cache_duration) - scanner._datetime_now() + if delta > timedelta(seconds=0): + logging.debug(_('next {name} cache update due in {time}').format( + name=self.filename, time=delta + )) + else: + raise SignatureDataOutdatedException() def fetch(self): try: - self.load_from_cache() - self.verify_data() - self.check_last_updated() - except ( - SignatureDataMalformedException, - SignatureDataVersionMismatchException, - SignatureDataOutdatedException - ): - try: - self.fetch_signatures_from_web() - except AttributeError: - # just load from defaults if fetch_signatures_from_web is not - # implemented - self.load_from_defaults() + self.fetch_signatures_from_web() self.write_to_cache() + except Exception as e: + raise Exception(_("downloading scanner signatures from '{}' failed").format(self.url)) from e def load(self): try: - self.load_from_cache() - self.verify_data() - except (SignatureDataMalformedException, SignatureDataVersionMismatchException): - self.load_from_defaults() + try: + self.load_from_cache() + self.verify_data() + self.check_last_updated() + except SignatureDataCacheMissException: + self.load_from_defaults() + except SignatureDataOutdatedException: + self.fetch_signatures_from_web() self.write_to_cache() + except (SignatureDataMalformedException, SignatureDataVersionMismatchException) as e: + logging.critical(_("scanner cache is malformed! You can clear it with: '{clear}'").format( + clear='rm -r {}'.format(common.get_config()['cachedir_scanner']) + )) + raise e def load_from_defaults(self): sig_file = (Path(__file__).parent / 'data' / 'scanner' / self.filename).resolve() with open(sig_file) as f: - self.data = json.load(f) + self.set_data(json.load(f)) def load_from_cache(self): sig_file = scanner._scanner_cachedir() / self.filename if not sig_file.exists(): - raise SignatureDataMalformedException() + raise SignatureDataCacheMissException() with open(sig_file) as f: - self.data = json.load(f) + self.set_data(json.load(f)) def write_to_cache(self): sig_file = scanner._scanner_cachedir() / self.filename @@ -231,29 +237,36 @@ class SignatureDataController: cleans and validates and cleans `self.data` ''' self.check_data_version() - valid_keys = ['timestamp', 'version', 'signatures'] + valid_keys = ['timestamp', 'last_updated', 'version', 'signatures', 'cache_duration'] for k in list(self.data.keys()): if k not in valid_keys: del self.data[k] + def set_data(self, new_data): + self.data = new_data + if 'cache_duration' in new_data: + self.cache_duration = timedelta(seconds=new_data['cache_duration']) + def fetch_signatures_from_web(self): logging.debug(_("downloading '{}'").format(self.url)) with urllib.request.urlopen(self.url) as f: - self.data = json.load(f) + self.set_data(json.load(f)) + self.data['last_updated'] = scanner._datetime_now().isoformat() class ExodusSignatureDataController(SignatureDataController): def __init__(self): super().__init__('Exodus signatures', 'exodus.yml', 'https://reports.exodus-privacy.eu.org/api/trackers') - self.cache_outdated_interval = timedelta(days=1) # refresh exodus cache after one day + self.cache_duration = timedelta(days=1) # refresh exodus cache after one day def fetch_signatures_from_web(self): logging.debug(_("downloading '{}'").format(self.url)) - self.data = { + data = { "signatures": {}, "timestamp": scanner._datetime_now().isoformat(), + "last_updated": scanner._datetime_now().isoformat(), "version": SCANNER_CACHE_VERSION, } @@ -261,7 +274,7 @@ class ExodusSignatureDataController(SignatureDataController): d = json.load(f) for tracker in d["trackers"].values(): if tracker.get('code_signature'): - self.data["signatures"][tracker["name"]] = { + data["signatures"][tracker["name"]] = { "name": tracker["name"], "warn_code_signatures": [tracker["code_signature"]], # exodus also provides network signatures, unused atm. @@ -273,6 +286,7 @@ class ExodusSignatureDataController(SignatureDataController): # etc. might be listed by exodus # too. } + self.set_data(data) class ScannerTool(): @@ -316,6 +330,7 @@ class ScannerTool(): def refresh(self): for sdc in self.sdcs: sdc.fetch_signatures_from_web() + sdc.write_to_cache() def add(self, new_controller: SignatureDataController): self.sdcs.append(new_controller) @@ -684,7 +699,7 @@ def main(): logging.getLogger().setLevel(logging.ERROR) # initialize/load configuration values - common.get_config(options) + common.get_config(opts=options) if options.refresh: scanner._get_tool().refresh() diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 65b70b60..32274396 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -502,39 +502,38 @@ class Test_SignatureDataController(unittest.TestCase): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') self.assertEqual(sdc.name, 'nnn') self.assertEqual(sdc.filename, 'fff.yml') - self.assertEqual(sdc.cache_outdated_interval, timedelta(999999)) + self.assertEqual(sdc.cache_duration, timedelta(999999)) self.assertDictEqual(sdc.data, {}) # check_last_updated def test_check_last_updated_ok(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['timestamp'] = datetime.now().astimezone().isoformat() + sdc.data['last_updated'] = datetime.now().astimezone().isoformat() sdc.check_last_updated() def test_check_last_updated_exception_cache_outdated(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.cache_outdated_interval = timedelta(days=7) - sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() + sdc.cache_duration = timedelta(days=7) + sdc.data['last_updated'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() - def test_check_last_updated_exception_missing_timestamp_value(self): - sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): - sdc.check_last_updated() - def test_check_last_updated_exception_not_string(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['timestamp'] = 12345 + sdc.data['last_updated'] = 12345 with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() def test_check_last_updated_exception_not_iso_formatted_string(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') - sdc.data['timestamp'] = '01/09/2002 10:11' + sdc.data['last_updated'] = '01/09/2002 10:11' with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException): sdc.check_last_updated() + def test_check_last_updated_no_exception_missing_when_last_updated_not_set(self): + sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json') + sdc.check_last_updated() + # check_data_version def test_check_data_version_ok(self): sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')