kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-22 12:10:11 +01:00
Code

Inject log_level into error_recorder

Browse Source 
format
This commit is contained in:
Grant Lanham 2024-08-25 01:03:28 -04:00
parent 9f7244d6f1
commit 118a748fba
3 changed files with 37 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
searx/metrics/error_recorder.py
View File

@ -6,6 +6,7 @@ import inspect
from json import JSONDecodeError from json import JSONDecodeError
from urllib.parse import urlparse from urllib.parse import urlparse
from httpx import HTTPError, HTTPStatusError from httpx import HTTPError, HTTPStatusError
import logging
from searx.exceptions import ( from searx.exceptions import (
SearxXPathSyntaxException, SearxXPathSyntaxException,
SearxEngineXPathException, SearxEngineXPathException,
@ -30,10 +31,20 @@ class ErrorContext: # pylint: disable=missing-class-docstring
'log_message', 'log_message',
'log_parameters', 'log_parameters',
'secondary', 'secondary',
'log_level',
) )
def __init__( # pylint: disable=too-many-arguments def __init__( # pylint: disable=too-many-arguments
self, filename, function, line_no, code, exception_classname, log_message, log_parameters, secondary self,
filename,
function,
line_no,
code,
exception_classname,
log_message,
log_parameters,
secondary,
log_level=logging.WARN,
): ):
self.filename = filename self.filename = filename
self.function = function self.function = function
@ -43,6 +54,7 @@ class ErrorContext: # pylint: disable=missing-class-docstring
self.log_message = log_message self.log_message = log_message
self.log_parameters = log_parameters self.log_parameters = log_parameters
self.secondary = secondary self.secondary = secondary
self.log_level: int = log_level
def __eq__(self, o) -> bool: # pylint: disable=invalid-name def __eq__(self, o) -> bool: # pylint: disable=invalid-name
if not isinstance(o, ErrorContext): if not isinstance(o, ErrorContext):
@ -56,6 +68,7 @@ class ErrorContext: # pylint: disable=missing-class-docstring
and self.log_message == o.log_message and self.log_message == o.log_message
and self.log_parameters == o.log_parameters and self.log_parameters == o.log_parameters
and self.secondary == o.secondary and self.secondary == o.secondary
and self.log_level == o.log_level
) )
def __hash__(self): def __hash__(self):
@ -69,11 +82,12 @@ class ErrorContext: # pylint: disable=missing-class-docstring
self.log_message, self.log_message,
self.log_parameters, self.log_parameters,
self.secondary, self.secondary,
self.log_level,
) )
) )
def __repr__(self): def __repr__(self):
return "ErrorContext({!r}, {!r}, {!r}, {!r}, {!r}, {!r}) {!r}".format( return "ErrorContext({!r}, {!r}, {!r}, {!r}, {!r}, {!r}), {!r}, {!r}".format(
self.filename, self.filename,
self.line_no, self.line_no,
self.code, self.code,
@ -81,13 +95,14 @@ class ErrorContext: # pylint: disable=missing-class-docstring
self.log_message, self.log_message,
self.log_parameters, self.log_parameters,
self.secondary, self.secondary,
self.log_level,
) )
def add_error_context(engine_name: str, error_context: ErrorContext) -> None: def add_error_context(engine_name: str, error_context: ErrorContext) -> None:
errors_for_engine = errors_per_engines.setdefault(engine_name, {}) errors_for_engine = errors_per_engines.setdefault(engine_name, {})
errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1 errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1
engines[engine_name].logger.warning('%s', str(error_context)) engines[engine_name].logger.log(error_context.log_level, '%s', str(error_context))
def get_trace(traces): def get_trace(traces):
@ -157,7 +172,9 @@ def get_exception_classname(exc: Exception) -> str:
return exc_module + '.' + exc_name return exc_module + '.' + exc_name
def get_error_context(framerecords, exception_classname, log_message, log_parameters, secondary) -> ErrorContext: def get_error_context(
framerecords, exception_classname, log_message, log_parameters, secondary, log_level: int
) -> ErrorContext:
searx_frame = get_trace(framerecords) searx_frame = get_trace(framerecords)
filename = searx_frame.filename filename = searx_frame.filename
if filename.startswith(searx_parent_dir): if filename.startswith(searx_parent_dir):
@ -166,30 +183,36 @@ def get_error_context(framerecords, exception_classname, log_message, log_parame
line_no = searx_frame.lineno line_no = searx_frame.lineno
code = searx_frame.code_context[0].strip() code = searx_frame.code_context[0].strip()
del framerecords del framerecords
return ErrorContext(filename, function, line_no, code, exception_classname, log_message, log_parameters, secondary) return ErrorContext(
filename, function, line_no, code, exception_classname, log_message, log_parameters, secondary, log_level
)
def count_exception(engine_name: str, exc: Exception, secondary: bool = False) -> None: def count_exception(engine_name: str, exc: Exception, secondary: bool = False, log_level=logging.WARN) -> None:
if not settings['general']['enable_metrics']: if not settings['general']['enable_metrics']:
return return
framerecords = inspect.trace() framerecords = inspect.trace()
try: try:
exception_classname = get_exception_classname(exc) exception_classname = get_exception_classname(exc)
log_parameters = get_messages(exc, framerecords[-1][1]) log_parameters = get_messages(exc, framerecords[-1][1])
error_context = get_error_context(framerecords, exception_classname, None, log_parameters, secondary) error_context = get_error_context(framerecords, exception_classname, None, log_parameters, secondary, log_level)
add_error_context(engine_name, error_context) add_error_context(engine_name, error_context)
finally: finally:
del framerecords del framerecords
def count_error( def count_error(
engine_name: str, log_message: str, log_parameters: typing.Optional[typing.Tuple] = None, secondary: bool = False engine_name: str,
log_message: str,
log_parameters: typing.Optional[typing.Tuple] = None,
secondary: bool = False,
log_level: int = logging.WARN,
) -> None: ) -> None:
if not settings['general']['enable_metrics']: if not settings['general']['enable_metrics']:
return return
framerecords = list(reversed(inspect.stack()[1:])) framerecords = list(reversed(inspect.stack()[1:]))
try: try:
error_context = get_error_context(framerecords, None, log_message, log_parameters or (), secondary) error_context = get_error_context(framerecords, None, log_message, log_parameters or (), secondary, log_level)
add_error_context(engine_name, error_context) add_error_context(engine_name, error_context)
finally: finally:
del framerecords del framerecords

6
searx/search/processors/abstract.py
View File

@ -3,6 +3,7 @@
""" """
import logging
from logging import Logger from logging import Logger
import threading import threading
from abc import abstractmethod, ABC from abc import abstractmethod, ABC
@ -96,10 +97,11 @@ class EngineProcessor(ABC):
result_container.add_unresponsive_engine(self.engine_name, error_message) result_container.add_unresponsive_engine(self.engine_name, error_message)
# metrics # metrics
counter_inc('engine', self.engine_name, 'search', 'count', 'error') counter_inc('engine', self.engine_name, 'search', 'count', 'error')
log_level = logging.WARN if self.engine_exc_info else logging.NOTSET
if isinstance(exception_or_message, BaseException): if isinstance(exception_or_message, BaseException):
count_exception(self.engine_name, exception_or_message) count_exception(self.engine_name, exception_or_message, log_level=log_level)
else: else:
count_error(self.engine_name, exception_or_message) count_error(self.engine_name, exception_or_message, log_level=log_level)
# suspend the engine ? # suspend the engine ?
if suspend: if suspend:
suspended_time = None suspended_time = None

1
searx/search/processors/online.py
View File

@ -126,6 +126,7 @@ class OnlineProcessor(EngineProcessor):
self.engine_name, self.engine_name,
'{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects), '{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects),
(status_code, reason, hostname), (status_code, reason, hostname),
self.engine_exc_info,
secondary=True, secondary=True,
) )