From 924f9afea37b6c545a03505a7ec291cf44654ca7 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 27 Apr 2021 15:13:39 +0200 Subject: [PATCH] [lint] pylint searx/search/processors files / BTW add some doc-strings Signed-off-by: Markus Heiser --- searx/search/processors/__init__.py | 32 ++++++++++++++------ searx/search/processors/abstract.py | 15 ++++++--- searx/search/processors/offline.py | 12 +++++--- searx/search/processors/online.py | 22 +++++++++----- searx/search/processors/online_currency.py | 15 +++++---- searx/search/processors/online_dictionary.py | 7 +++-- 6 files changed, 68 insertions(+), 35 deletions(-) diff --git a/searx/search/processors/__init__.py b/searx/search/processors/__init__.py index 4cae3cd0f..caac74e65 100644 --- a/searx/search/processors/__init__.py +++ b/searx/search/processors/__init__.py @@ -1,37 +1,49 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint + +"""Implement request processores used by engine-types. + +""" + +__all__ = [ + 'EngineProcessor', + 'OfflineProcessor', + 'OnlineProcessor', + 'OnlineDictionaryProcessor', + 'OnlineCurrencyProcessor', + 'processors', +] + +from searx import logger +import searx.engines as engines from .online import OnlineProcessor from .offline import OfflineProcessor from .online_dictionary import OnlineDictionaryProcessor from .online_currency import OnlineCurrencyProcessor from .abstract import EngineProcessor -from searx import logger -import searx.engines as engines - -__all__ = ['EngineProcessor', 'OfflineProcessor', 'OnlineProcessor', - 'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'processors'] logger = logger.getChild('search.processors') processors = {} - +"""Cache request processores, stored by *engine-name* (:py:func:`initialize`)""" def get_processor_class(engine_type): + """Return processor class according to the ``engine_type``""" for c in [OnlineProcessor, OfflineProcessor, OnlineDictionaryProcessor, OnlineCurrencyProcessor]: if c.engine_type == engine_type: return c return None - def get_processor(engine, engine_name): + """Return processor instance that fits to ``engine.engine.type``)""" engine_type = getattr(engine, 'engine_type', 'online') processor_class = get_processor_class(engine_type) if processor_class: return processor_class(engine, engine_name) - else: - return None - + return None def initialize(engine_list): + """Initialize all engines and store a processor for each engine in :py:obj:`processors`.""" engines.initialize_engines(engine_list) for engine_name, engine in engines.engines.items(): processor = get_processor(engine, engine_name) diff --git a/searx/search/processors/abstract.py b/searx/search/processors/abstract.py index 2a36222d4..38811d87c 100644 --- a/searx/search/processors/abstract.py +++ b/searx/search/processors/abstract.py @@ -1,4 +1,9 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint + +"""Abstract base classes for engine request processores. + +""" import threading from abc import abstractmethod, ABC @@ -10,12 +15,13 @@ from searx.network import get_time_for_thread, get_network from searx.metrics import histogram_observe, counter_inc, count_exception, count_error from searx.exceptions import SearxEngineAccessDeniedException - logger = logger.getChild('searx.search.processor') SUSPENDED_STATUS = {} +# pylint: disable=missing-function-docstring class SuspendedStatus: + """Class to handle suspend state.""" __slots__ = 'suspend_end_time', 'suspend_reason', 'continuous_errors', 'lock' @@ -49,6 +55,7 @@ class SuspendedStatus: class EngineProcessor(ABC): + """Base classes used for all types of reqest processores.""" __slots__ = 'engine', 'engine_name', 'lock', 'suspended_status' @@ -143,9 +150,7 @@ class EngineProcessor(ABC): if tests is None: tests = getattr(self.engine, 'additional_tests', {}) tests.update(self.get_default_tests()) - return tests - else: - return tests + return tests - def get_default_tests(self): + def get_default_tests(self): # pylint: disable=no-self-use return {} diff --git a/searx/search/processors/offline.py b/searx/search/processors/offline.py index ad03fed4b..f40626f39 100644 --- a/searx/search/processors/offline.py +++ b/searx/search/processors/offline.py @@ -1,13 +1,17 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint + +"""Processores for engine-type: ``offline`` + +""" from searx import logger -from searx.search.processors.abstract import EngineProcessor - +from .abstract import EngineProcessor logger = logger.getChild('searx.search.processor.offline') - class OfflineProcessor(EngineProcessor): + """Processor class used by ``offline`` engines""" engine_type = 'offline' @@ -21,6 +25,6 @@ class OfflineProcessor(EngineProcessor): except ValueError as e: # do not record the error logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e)) - except Exception as e: + except Exception as e: # pylint: disable=broad-except self.handle_exception(result_container, e) logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index 57422c007..93a9c6cbf 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -1,24 +1,29 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint + +"""Processores for engine-type: ``online`` + +""" from time import time import asyncio - import httpx import searx.network from searx import logger from searx.utils import gen_useragent -from searx.exceptions import (SearxEngineAccessDeniedException, SearxEngineCaptchaException, - SearxEngineTooManyRequestsException,) +from searx.exceptions import ( + SearxEngineAccessDeniedException, + SearxEngineCaptchaException, + SearxEngineTooManyRequestsException, +) from searx.metrics.error_recorder import count_error - -from searx.search.processors.abstract import EngineProcessor - +from .abstract import EngineProcessor logger = logger.getChild('searx.search.processor.online') - def default_request_params(): + """Default request parameters for ``online`` engines.""" return { 'method': 'GET', 'headers': {}, @@ -31,6 +36,7 @@ def default_request_params(): class OnlineProcessor(EngineProcessor): + """Processor class for ``online`` engines.""" engine_type = 'online' @@ -153,7 +159,7 @@ class OnlineProcessor(EngineProcessor): except SearxEngineAccessDeniedException as e: self.handle_exception(result_container, e, suspend=True) logger.exception('engine {0} : Searx is blocked'.format(self.engine_name)) - except Exception as e: + except Exception as e: # pylint: disable=broad-except self.handle_exception(result_container, e) logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) diff --git a/searx/search/processors/online_currency.py b/searx/search/processors/online_currency.py index 0dc3f3b6a..4f642fa72 100644 --- a/searx/search/processors/online_currency.py +++ b/searx/search/processors/online_currency.py @@ -1,4 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Processores for engine-type: ``online_currency`` + +""" import unicodedata import re @@ -6,32 +10,31 @@ import re from searx.data import CURRENCIES from .online import OnlineProcessor - parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I) +# pylint: disable=missing-function-docstring def normalize_name(name): name = name.lower().replace('-', ' ').rstrip('s') name = re.sub(' +', ' ', name) return unicodedata.normalize('NFKD', name).lower() - def name_to_iso4217(name): - global CURRENCIES + global CURRENCIES # pylint: disable=global-statement name = normalize_name(name) currency = CURRENCIES['names'].get(name, [name]) if isinstance(currency, str): return currency return currency[0] - def iso4217_to_name(iso4217, language): - global CURRENCIES + global CURRENCIES # pylint: disable=global-statement return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217) - class OnlineCurrencyProcessor(OnlineProcessor): + """Processor class used by ``online_currency`` engines.""" + engine_type = 'online_currency' def get_params(self, search_query, engine_category): diff --git a/searx/search/processors/online_dictionary.py b/searx/search/processors/online_dictionary.py index 987c710a1..11ca0335d 100644 --- a/searx/search/processors/online_dictionary.py +++ b/searx/search/processors/online_dictionary.py @@ -1,15 +1,18 @@ # SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Processores for engine-type: ``online_dictionary`` + +""" import re from searx.utils import is_valid_lang from .online import OnlineProcessor - parser_re = re.compile('.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I) - class OnlineDictionaryProcessor(OnlineProcessor): + """Processor class used by ``online_dictionnary`` engines.""" engine_type = 'online_dictionnary'