kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-10-01 09:00:15 +02:00
Code

[mod] searx.metrics & searx.search: use the engine loggers

Browse Source 
metrics & processors use the engine logger
This commit is contained in:
Alexandre Flament 2021-09-06 19:46:08 +02:00
parent 76e0f6807c
commit b513917ef9
6 changed files with 35 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
searx/metrics/error_recorder.py
View File

@ -5,7 +5,8 @@ from urllib.parse import urlparse
from httpx import HTTPError, HTTPStatusError from httpx import HTTPError, HTTPStatusError
from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException, from searx.exceptions import (SearxXPathSyntaxException, SearxEngineXPathException, SearxEngineAPIException,
SearxEngineAccessDeniedException) SearxEngineAccessDeniedException)
from searx import logger, searx_parent_dir from searx import searx_parent_dir
from searx.engines import engines
errors_per_engines = {} errors_per_engines = {}
@ -47,7 +48,7 @@ class ErrorContext:
def add_error_context(engine_name: str, error_context: ErrorContext) -> None: def add_error_context(engine_name: str, error_context: ErrorContext) -> None:
errors_for_engine = errors_per_engines.setdefault(engine_name, {}) errors_for_engine = errors_per_engines.setdefault(engine_name, {})
errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1 errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1
logger.debug('%s: %s', engine_name, str(error_context)) engines[engine_name].logger.warning('%s', str(error_context))
def get_trace(traces): def get_trace(traces):

2
searx/search/__init__.py
View File

@ -147,7 +147,7 @@ class Search:
if th.is_alive(): if th.is_alive():
th._timeout = True th._timeout = True
self.result_container.add_unresponsive_engine(th._engine_name, 'timeout') self.result_container.add_unresponsive_engine(th._engine_name, 'timeout')
logger.warning('engine timeout: {0}'.format(th._engine_name)) PROCESSORS[th._engine_name].logger.error('engine timeout')
def search_standard(self): def search_standard(self):
""" """

2
searx/search/processors/__init__.py
View File

@ -65,6 +65,6 @@ def initialize(engine_list):
processor = get_processor(engine, engine_name) processor = get_processor(engine, engine_name)
initialize_processor(processor) initialize_processor(processor)
if processor is None: if processor is None:
logger.error('Error get processor for engine %s', engine_name) engine.logger.error('Error get processor for engine %s', engine_name)
else: else:
PROCESSORS[engine_name] = processor PROCESSORS[engine_name] = processor

15
searx/search/processors/abstract.py
View File

@ -9,8 +9,8 @@ import threading
from abc import abstractmethod, ABC from abc import abstractmethod, ABC
from timeit import default_timer from timeit import default_timer
from searx import logger from searx import settings, logger
from searx.engines import settings from searx.engines import engines
from searx.network import get_time_for_thread, get_network from searx.network import get_time_for_thread, get_network
from searx.metrics import histogram_observe, counter_inc, count_exception, count_error from searx.metrics import histogram_observe, counter_inc, count_exception, count_error
from searx.exceptions import SearxEngineAccessDeniedException, SearxEngineResponseException from searx.exceptions import SearxEngineAccessDeniedException, SearxEngineResponseException
@ -43,7 +43,7 @@ class SuspendedStatus:
self.continuous_errors * settings['search']['ban_time_on_fail']) self.continuous_errors * settings['search']['ban_time_on_fail'])
self.suspend_end_time = default_timer() + suspended_time self.suspend_end_time = default_timer() + suspended_time
self.suspend_reason = suspend_reason self.suspend_reason = suspend_reason
logger.debug('Suspend engine for %i seconds', suspended_time) logger.debug('Suspend for %i seconds', suspended_time)
def resume(self): def resume(self):
with self.lock: with self.lock:
@ -56,11 +56,12 @@ class SuspendedStatus:
class EngineProcessor(ABC): class EngineProcessor(ABC):
"""Base classes used for all types of reqest processores.""" """Base classes used for all types of reqest processores."""
__slots__ = 'engine', 'engine_name', 'lock', 'suspended_status' __slots__ = 'engine', 'engine_name', 'lock', 'suspended_status', 'logger'
def __init__(self, engine, engine_name): def __init__(self, engine, engine_name):
self.engine = engine self.engine = engine
self.engine_name = engine_name self.engine_name = engine_name
self.logger = engines[engine_name].logger
key = get_network(self.engine_name) key = get_network(self.engine_name)
key = id(key) if key else self.engine_name key = id(key) if key else self.engine_name
self.suspended_status = SUSPENDED_STATUS.setdefault(key, SuspendedStatus()) self.suspended_status = SUSPENDED_STATUS.setdefault(key, SuspendedStatus())
@ -69,11 +70,11 @@ class EngineProcessor(ABC):
try: try:
self.engine.init(get_engine_from_settings(self.engine_name)) self.engine.init(get_engine_from_settings(self.engine_name))
except SearxEngineResponseException as exc: except SearxEngineResponseException as exc:
logger.warn('%s engine: Fail to initialize // %s', self.engine_name, exc) self.logger.warn('Fail to initialize // %s', exc)
except Exception: # pylint: disable=broad-except except Exception: # pylint: disable=broad-except
logger.exception('%s engine: Fail to initialize', self.engine_name) self.logger.exception('Fail to initialize')
else: else:
logger.debug('%s engine: Initialized', self.engine_name) self.logger.debug('Initialized')
@property @property
def has_initialize_function(self): def has_initialize_function(self):

6
searx/search/processors/offline.py
View File

@ -5,10 +5,8 @@
""" """
from searx import logger
from .abstract import EngineProcessor from .abstract import EngineProcessor
logger = logger.getChild('searx.search.processor.offline')
class OfflineProcessor(EngineProcessor): class OfflineProcessor(EngineProcessor):
"""Processor class used by ``offline`` engines""" """Processor class used by ``offline`` engines"""
@ -24,7 +22,7 @@ class OfflineProcessor(EngineProcessor):
self.extend_container(result_container, start_time, search_results) self.extend_container(result_container, start_time, search_results)
except ValueError as e: except ValueError as e:
# do not record the error # do not record the error
logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e)) self.logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e))
except Exception as e: # pylint: disable=broad-except except Exception as e: # pylint: disable=broad-except
self.handle_exception(result_container, e) self.handle_exception(result_container, e)
logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) self.logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e))

36
searx/search/processors/online.py
View File

@ -10,7 +10,6 @@ import asyncio
import httpx import httpx
import searx.network import searx.network
from searx import logger
from searx.utils import gen_useragent from searx.utils import gen_useragent
from searx.exceptions import ( from searx.exceptions import (
SearxEngineAccessDeniedException, SearxEngineAccessDeniedException,
@ -20,7 +19,6 @@ from searx.exceptions import (
from searx.metrics.error_recorder import count_error from searx.metrics.error_recorder import count_error
from .abstract import EngineProcessor from .abstract import EngineProcessor
logger = logger.getChild('searx.search.processor.online')
def default_request_params(): def default_request_params():
"""Default request parameters for ``online`` engines.""" """Default request parameters for ``online`` engines."""
@ -146,31 +144,37 @@ class OnlineProcessor(EngineProcessor):
except (httpx.TimeoutException, asyncio.TimeoutError) as e: except (httpx.TimeoutException, asyncio.TimeoutError) as e:
# requests timeout (connect or read) # requests timeout (connect or read)
self.handle_exception(result_container, e, suspend=True) self.handle_exception(result_container, e, suspend=True)
logger.error("engine {0} : HTTP requests timeout" self.logger.error(
"(search duration : {1} s, timeout: {2} s) : {3}" "HTTP requests timeout (search duration : {0} s, timeout: {1} s) : {2}"
.format(self.engine_name, default_timer() - start_time, .format(
timeout_limit, default_timer() - start_time,
e.__class__.__name__)) timeout_limit,
e.__class__.__name__
)
)
except (httpx.HTTPError, httpx.StreamError) as e: except (httpx.HTTPError, httpx.StreamError) as e:
# other requests exception # other requests exception
self.handle_exception(result_container, e, suspend=True) self.handle_exception(result_container, e, suspend=True)
logger.exception("engine {0} : requests exception" self.logger.exception(
"(search duration : {1} s, timeout: {2} s) : {3}" "requests exception (search duration : {0} s, timeout: {1} s) : {2}"
.format(self.engine_name, default_timer() - start_time, .format(
timeout_limit, default_timer() - start_time,
e)) timeout_limit,
e
)
)
except SearxEngineCaptchaException as e: except SearxEngineCaptchaException as e:
self.handle_exception(result_container, e, suspend=True) self.handle_exception(result_container, e, suspend=True)
logger.exception('engine {0} : CAPTCHA'.format(self.engine_name)) self.logger.exception('CAPTCHA')
except SearxEngineTooManyRequestsException as e: except SearxEngineTooManyRequestsException as e:
self.handle_exception(result_container, e, suspend=True) self.handle_exception(result_container, e, suspend=True)
logger.exception('engine {0} : Too many requests'.format(self.engine_name)) self.logger.exception('Too many requests')
except SearxEngineAccessDeniedException as e: except SearxEngineAccessDeniedException as e:
self.handle_exception(result_container, e, suspend=True) self.handle_exception(result_container, e, suspend=True)
logger.exception('engine {0} : Searx is blocked'.format(self.engine_name)) self.logger.exception('Searx is blocked')
except Exception as e: # pylint: disable=broad-except except Exception as e: # pylint: disable=broad-except
self.handle_exception(result_container, e) self.handle_exception(result_container, e)
logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e)) self.logger.exception('exception : {0}'.format(e))
def get_default_tests(self): def get_default_tests(self):
tests = {} tests = {}