From b513917ef94d2f18e6db3228363f8089ce1ba08a Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Mon, 6 Sep 2021 19:46:08 +0200 Subject: [PATCH] [mod] searx.metrics & searx.search: use the engine loggers metrics & processors use the engine logger --- searx/metrics/error_recorder.py | 5 ++-- searx/search/__init__.py | 2 +- searx/search/processors/__init__.py | 2 +- searx/search/processors/abstract.py | 15 ++++++------ searx/search/processors/offline.py | 6 ++--- searx/search/processors/online.py | 36 ++++++++++++++++------------- 6 files changed, 35 insertions(+), 31 deletions(-) diff --git a/searx/metrics/error_recorder.py b/searx/metrics/error_recorder.py index c5de008cc..6963cda2f 100644 --- a/searx/metrics/error_recorder.py +++ b/searx/metrics/error_recorder.py @@ -5,7 +5,8 @@ from urllib.parse import urlparse from httpx import HTTPError, HTTPStatusError from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException, SearxEngineAccessDeniedException) -from searx import logger, searx_parent_dir +from searx import searx_parent_dir +from searx.engines import engines errors_per_engines = {} @@ -47,7 +48,7 @@ class ErrorContext: def add_error_context(engine_name: str, error_context: ErrorContext) -> None: errors_for_engine = errors_per_engines.setdefault(engine_name, {}) errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1 - logger.debug('%s: %s', engine_name, str(error_context)) + engines[engine_name].logger.warning('%s', str(error_context)) def get_trace(traces): diff --git a/searx/search/__init__.py b/searx/search/__init__.py index 69d7ffb25..97515622d 100644 --- a/searx/search/__init__.py +++ b/searx/search/__init__.py @@ -147,7 +147,7 @@ class Search: if th.is_alive(): th._timeout = True self.result_container.add_unresponsive_engine(th._engine_name, 'timeout') - logger.warning('engine timeout: {0}'.format(th._engine_name)) + PROCESSORS[th._engine_name].logger.error('engine timeout') def search_standard(self): """ diff --git a/searx/search/processors/__init__.py b/searx/search/processors/__init__.py index b3d121f0e..8108f8dfa 100644 --- a/searx/search/processors/__init__.py +++ b/searx/search/processors/__init__.py @@ -65,6 +65,6 @@ def initialize(engine_list): processor = get_processor(engine, engine_name) initialize_processor(processor) if processor is None: - logger.error('Error get processor for engine %s', engine_name) + engine.logger.error('Error get processor for engine %s', engine_name) else: PROCESSORS[engine_name] = processor diff --git a/searx/search/processors/abstract.py b/searx/search/processors/abstract.py index c8d81c026..b5fa063fd 100644 --- a/searx/search/processors/abstract.py +++ b/searx/search/processors/abstract.py @@ -9,8 +9,8 @@ import threading from abc import abstractmethod, ABC from timeit import default_timer -from searx import logger -from searx.engines import settings +from searx import settings, logger +from searx.engines import engines from searx.network import get_time_for_thread, get_network from searx.metrics import histogram_observe, counter_inc, count_exception, count_error from searx.exceptions import SearxEngineAccessDeniedException, SearxEngineResponseException @@ -43,7 +43,7 @@ class SuspendedStatus: self.continuous_errors * settings['search']['ban_time_on_fail']) self.suspend_end_time = default_timer() + suspended_time self.suspend_reason = suspend_reason - logger.debug('Suspend engine for %i seconds', suspended_time) + logger.debug('Suspend for %i seconds', suspended_time) def resume(self): with self.lock: @@ -56,11 +56,12 @@ class SuspendedStatus: class EngineProcessor(ABC): """Base classes used for all types of reqest processores.""" - __slots__ = 'engine', 'engine_name', 'lock', 'suspended_status' + __slots__ = 'engine', 'engine_name', 'lock', 'suspended_status', 'logger' def __init__(self, engine, engine_name): self.engine = engine self.engine_name = engine_name + self.logger = engines[engine_name].logger key = get_network(self.engine_name) key = id(key) if key else self.engine_name self.suspended_status = SUSPENDED_STATUS.setdefault(key, SuspendedStatus()) @@ -69,11 +70,11 @@ class EngineProcessor(ABC): try: self.engine.init(get_engine_from_settings(self.engine_name)) except SearxEngineResponseException as exc: - logger.warn('%s engine: Fail to initialize // %s', self.engine_name, exc) + self.logger.warn('Fail to initialize // %s', exc) except Exception: # pylint: disable=broad-except - logger.exception('%s engine: Fail to initialize', self.engine_name) + self.logger.exception('Fail to initialize') else: - logger.debug('%s engine: Initialized', self.engine_name) + self.logger.debug('Initialized') @property def has_initialize_function(self): diff --git a/searx/search/processors/offline.py b/searx/search/processors/offline.py index f40626f39..ec7a4a36e 100644 --- a/searx/search/processors/offline.py +++ b/searx/search/processors/offline.py @@ -5,10 +5,8 @@ """ -from searx import logger from .abstract import EngineProcessor -logger = logger.getChild('searx.search.processor.offline') class OfflineProcessor(EngineProcessor): """Processor class used by ``offline`` engines""" @@ -24,7 +22,7 @@ class OfflineProcessor(EngineProcessor): self.extend_container(result_container, start_time, search_results) except ValueError as e: # do not record the error - logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e)) + self.logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e)) except Exception as e: # pylint: disable=broad-except self.handle_exception(result_container, e) - logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) + self.logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index 48a514e8a..c4ee58e11 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -10,7 +10,6 @@ import asyncio import httpx import searx.network -from searx import logger from searx.utils import gen_useragent from searx.exceptions import ( SearxEngineAccessDeniedException, @@ -20,7 +19,6 @@ from searx.exceptions import ( from searx.metrics.error_recorder import count_error from .abstract import EngineProcessor -logger = logger.getChild('searx.search.processor.online') def default_request_params(): """Default request parameters for ``online`` engines.""" @@ -146,31 +144,37 @@ class OnlineProcessor(EngineProcessor): except (httpx.TimeoutException, asyncio.TimeoutError) as e: # requests timeout (connect or read) self.handle_exception(result_container, e, suspend=True) - logger.error("engine {0} : HTTP requests timeout" - "(search duration : {1} s, timeout: {2} s) : {3}" - .format(self.engine_name, default_timer() - start_time, - timeout_limit, - e.__class__.__name__)) + self.logger.error( + "HTTP requests timeout (search duration : {0} s, timeout: {1} s) : {2}" + .format( + default_timer() - start_time, + timeout_limit, + e.__class__.__name__ + ) + ) except (httpx.HTTPError, httpx.StreamError) as e: # other requests exception self.handle_exception(result_container, e, suspend=True) - logger.exception("engine {0} : requests exception" - "(search duration : {1} s, timeout: {2} s) : {3}" - .format(self.engine_name, default_timer() - start_time, - timeout_limit, - e)) + self.logger.exception( + "requests exception (search duration : {0} s, timeout: {1} s) : {2}" + .format( + default_timer() - start_time, + timeout_limit, + e + ) + ) except SearxEngineCaptchaException as e: self.handle_exception(result_container, e, suspend=True) - logger.exception('engine {0} : CAPTCHA'.format(self.engine_name)) + self.logger.exception('CAPTCHA') except SearxEngineTooManyRequestsException as e: self.handle_exception(result_container, e, suspend=True) - logger.exception('engine {0} : Too many requests'.format(self.engine_name)) + self.logger.exception('Too many requests') except SearxEngineAccessDeniedException as e: self.handle_exception(result_container, e, suspend=True) - logger.exception('engine {0} : Searx is blocked'.format(self.engine_name)) + self.logger.exception('Searx is blocked') except Exception as e: # pylint: disable=broad-except self.handle_exception(result_container, e) - logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) + self.logger.exception('exception : {0}'.format(e)) def get_default_tests(self): tests = {}