mirror of
https://github.com/searxng/searxng.git
synced 2024-06-16 12:10:13 +02:00
[mod] isolation of botdetection from the limiter
This patch was inspired by the discussion around PR-2882 [2]. The goals of this patch are: 1. Convert plugin searx.plugin.limiter to normal code [1] 2. isolation of botdetection from the limiter [2] 3. searx/{tools => botdetection}/config.py and drop searx.tools 4. in URL /config, 'limiter.enabled' is true only if the limiter is really enabled (Redis is available). This patch moves all the code that belongs to botdetection into namespace searx.botdetection and code that belongs to limiter is placed in namespace searx.limiter. Tthe limiter used to be a plugin at some point botdetection was added, it was not a plugin. The modularization of these two components was long overdue. With the clear modularization, the documentation could then also be organized according to the architecture. [1] https://github.com/searxng/searxng/pull/2882 [2] https://github.com/searxng/searxng/pull/2882#issuecomment-1741716891 To test: - check the app works without the limiter, check `/config` - check the app works with the limiter and with the token, check `/config` - make docs.live .. and read - http://0.0.0.0:8000/admin/searx.limiter.html - http://0.0.0.0:8000/src/searx.botdetection.html#botdetection Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
parent
b05a15540e
commit
fd814aac86
|
@ -15,7 +15,7 @@ Administrator documentation
|
||||||
installation-apache
|
installation-apache
|
||||||
update-searxng
|
update-searxng
|
||||||
answer-captcha
|
answer-captcha
|
||||||
searx.botdetection
|
searx.limiter
|
||||||
api
|
api
|
||||||
architecture
|
architecture
|
||||||
plugins
|
plugins
|
||||||
|
|
17
docs/admin/searx.limiter.rst
Normal file
17
docs/admin/searx.limiter.rst
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
.. _limiter:
|
||||||
|
|
||||||
|
=======
|
||||||
|
Limiter
|
||||||
|
=======
|
||||||
|
|
||||||
|
.. sidebar:: info
|
||||||
|
|
||||||
|
The limiter requires a :ref:`Redis <settings redis>` database.
|
||||||
|
|
||||||
|
.. contents::
|
||||||
|
:depth: 2
|
||||||
|
:local:
|
||||||
|
:backlinks: entry
|
||||||
|
|
||||||
|
.. automodule:: searx.limiter
|
||||||
|
:members:
|
|
@ -36,11 +36,9 @@
|
||||||
``secret_key`` : ``$SEARXNG_SECRET``
|
``secret_key`` : ``$SEARXNG_SECRET``
|
||||||
Used for cryptography purpose.
|
Used for cryptography purpose.
|
||||||
|
|
||||||
.. _limiter:
|
|
||||||
|
|
||||||
``limiter`` :
|
``limiter`` :
|
||||||
Rate limit the number of request on the instance, block some bots. The
|
Rate limit the number of request on the instance, block some bots. The
|
||||||
:ref:`limiter src` requires a :ref:`settings redis` database.
|
:ref:`limiter` requires a :ref:`settings redis` database.
|
||||||
|
|
||||||
.. _image_proxy:
|
.. _image_proxy:
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
import sys, os
|
import sys, os
|
||||||
|
from pathlib import Path
|
||||||
from pallets_sphinx_themes import ProjectLink
|
from pallets_sphinx_themes import ProjectLink
|
||||||
|
|
||||||
from searx import get_setting
|
from searx import get_setting
|
||||||
|
@ -13,7 +14,6 @@ project = 'SearXNG'
|
||||||
copyright = 'SearXNG team'
|
copyright = 'SearXNG team'
|
||||||
author = 'SearXNG team'
|
author = 'SearXNG team'
|
||||||
release, version = VERSION_STRING, VERSION_STRING
|
release, version = VERSION_STRING, VERSION_STRING
|
||||||
|
|
||||||
SEARXNG_URL = get_setting('server.base_url') or 'https://example.org/searxng'
|
SEARXNG_URL = get_setting('server.base_url') or 'https://example.org/searxng'
|
||||||
ISSUE_URL = get_setting('brand.issue_url')
|
ISSUE_URL = get_setting('brand.issue_url')
|
||||||
DOCS_URL = get_setting('brand.docs_url')
|
DOCS_URL = get_setting('brand.docs_url')
|
||||||
|
@ -22,6 +22,9 @@ PRIVACYPOLICY_URL = get_setting('general.privacypolicy_url')
|
||||||
CONTACT_URL = get_setting('general.contact_url')
|
CONTACT_URL = get_setting('general.contact_url')
|
||||||
WIKI_URL = get_setting('brand.wiki_url')
|
WIKI_URL = get_setting('brand.wiki_url')
|
||||||
|
|
||||||
|
SOURCEDIR = Path(__file__).parent.parent / "searx"
|
||||||
|
os.environ['SOURCEDIR'] = str(SOURCEDIR)
|
||||||
|
|
||||||
# hint: sphinx.ext.viewcode won't highlight when 'highlight_language' [1] is set
|
# hint: sphinx.ext.viewcode won't highlight when 'highlight_language' [1] is set
|
||||||
# to string 'none' [2]
|
# to string 'none' [2]
|
||||||
#
|
#
|
||||||
|
|
|
@ -12,8 +12,10 @@ Bot Detection
|
||||||
.. automodule:: searx.botdetection
|
.. automodule:: searx.botdetection
|
||||||
:members:
|
:members:
|
||||||
|
|
||||||
.. automodule:: searx.botdetection.limiter
|
.. _botdetection ip_lists:
|
||||||
:members:
|
|
||||||
|
IP lists
|
||||||
|
========
|
||||||
|
|
||||||
.. automodule:: searx.botdetection.ip_lists
|
.. automodule:: searx.botdetection.ip_lists
|
||||||
:members:
|
:members:
|
||||||
|
@ -50,3 +52,11 @@ Probe HTTP headers
|
||||||
|
|
||||||
.. automodule:: searx.botdetection.http_user_agent
|
.. automodule:: searx.botdetection.http_user_agent
|
||||||
:members:
|
:members:
|
||||||
|
|
||||||
|
.. _botdetection config:
|
||||||
|
|
||||||
|
Config
|
||||||
|
======
|
||||||
|
|
||||||
|
.. automodule:: searx.botdetection.config
|
||||||
|
:members:
|
|
@ -2,43 +2,22 @@
|
||||||
# lint: pylint
|
# lint: pylint
|
||||||
""".. _botdetection src:
|
""".. _botdetection src:
|
||||||
|
|
||||||
The :ref:`limiter <limiter src>` implements several methods to block bots:
|
Implementations used for bot detection.
|
||||||
|
|
||||||
a. Analysis of the HTTP header in the request / can be easily bypassed.
|
|
||||||
|
|
||||||
b. Block and pass lists in which IPs are listed / difficult to maintain, since
|
|
||||||
the IPs of bots are not all known and change over the time.
|
|
||||||
|
|
||||||
c. Detection of bots based on the behavior of the requests and blocking and, if
|
|
||||||
necessary, unblocking of the IPs via a dynamically changeable IP block list.
|
|
||||||
|
|
||||||
For dynamically changeable IP lists a Redis database is needed and for any kind
|
|
||||||
of IP list the determination of the IP of the client is essential. The IP of
|
|
||||||
the client is determined via the X-Forwarded-For_ HTTP header
|
|
||||||
|
|
||||||
.. _X-Forwarded-For:
|
|
||||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
|
|
||||||
|
|
||||||
X-Forwarded-For
|
|
||||||
===============
|
|
||||||
|
|
||||||
.. attention::
|
|
||||||
|
|
||||||
A correct setup of the HTTP request headers ``X-Forwarded-For`` and
|
|
||||||
``X-Real-IP`` is essential to be able to assign a request to an IP correctly:
|
|
||||||
|
|
||||||
- `NGINX RequestHeader`_
|
|
||||||
- `Apache RequestHeader`_
|
|
||||||
|
|
||||||
.. _NGINX RequestHeader:
|
|
||||||
https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site
|
|
||||||
.. _Apache RequestHeader:
|
|
||||||
https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site
|
|
||||||
|
|
||||||
.. autofunction:: searx.botdetection.get_real_ip
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from ._helpers import dump_request
|
from ._helpers import dump_request
|
||||||
from ._helpers import get_real_ip
|
from ._helpers import get_real_ip
|
||||||
|
from ._helpers import get_network
|
||||||
from ._helpers import too_many_requests
|
from ._helpers import too_many_requests
|
||||||
|
|
||||||
|
__all__ = ['dump_request', 'get_network', 'get_real_ip', 'too_many_requests']
|
||||||
|
|
||||||
|
redis_client = None
|
||||||
|
cfg = None
|
||||||
|
|
||||||
|
|
||||||
|
def init(_cfg, _redis_client):
|
||||||
|
global redis_client, cfg # pylint: disable=global-statement
|
||||||
|
redis_client = _redis_client
|
||||||
|
cfg = _cfg
|
||||||
|
|
|
@ -13,8 +13,8 @@ from ipaddress import (
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
|
|
||||||
from searx.tools import config
|
|
||||||
from searx import logger
|
from searx import logger
|
||||||
|
from . import config
|
||||||
|
|
||||||
logger = logger.getChild('botdetection')
|
logger = logger.getChild('botdetection')
|
||||||
|
|
||||||
|
@ -104,10 +104,10 @@ def get_real_ip(request: flask.Request) -> str:
|
||||||
if not forwarded_for:
|
if not forwarded_for:
|
||||||
_log_error_only_once("X-Forwarded-For header is not set!")
|
_log_error_only_once("X-Forwarded-For header is not set!")
|
||||||
else:
|
else:
|
||||||
from .limiter import get_cfg # pylint: disable=import-outside-toplevel, cyclic-import
|
from . import cfg # pylint: disable=import-outside-toplevel, cyclic-import
|
||||||
|
|
||||||
forwarded_for = [x.strip() for x in forwarded_for.split(',')]
|
forwarded_for = [x.strip() for x in forwarded_for.split(',')]
|
||||||
x_for: int = get_cfg()['real_ip.x_for'] # type: ignore
|
x_for: int = cfg['real_ip.x_for'] # type: ignore
|
||||||
forwarded_for = forwarded_for[-min(len(forwarded_for), x_for)]
|
forwarded_for = forwarded_for[-min(len(forwarded_for), x_for)]
|
||||||
|
|
||||||
if not real_ip:
|
if not real_ip:
|
||||||
|
|
|
@ -24,7 +24,7 @@ from ipaddress import (
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
|
|
||||||
from searx.tools import config
|
from . import config
|
||||||
from ._helpers import too_many_requests
|
from ._helpers import too_many_requests
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -25,7 +25,7 @@ from ipaddress import (
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
|
|
||||||
from searx.tools import config
|
from . import config
|
||||||
from ._helpers import too_many_requests
|
from ._helpers import too_many_requests
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -21,7 +21,7 @@ from ipaddress import (
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
|
|
||||||
from searx.tools import config
|
from . import config
|
||||||
from ._helpers import too_many_requests
|
from ._helpers import too_many_requests
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,7 @@ from ipaddress import (
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
|
|
||||||
from searx.tools import config
|
from . import config
|
||||||
from ._helpers import too_many_requests
|
from ._helpers import too_many_requests
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,7 @@ from ipaddress import (
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
|
|
||||||
from searx.tools import config
|
from . import config
|
||||||
from ._helpers import too_many_requests
|
from ._helpers import too_many_requests
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -13,8 +13,7 @@ and at least for a maximum of 10 minutes.
|
||||||
|
|
||||||
The :py:obj:`.link_token` method can be used to investigate whether a request is
|
The :py:obj:`.link_token` method can be used to investigate whether a request is
|
||||||
*suspicious*. To activate the :py:obj:`.link_token` method in the
|
*suspicious*. To activate the :py:obj:`.link_token` method in the
|
||||||
:py:obj:`.ip_limit` method add the following to your
|
:py:obj:`.ip_limit` method add the following configuration:
|
||||||
``/etc/searxng/limiter.toml``:
|
|
||||||
|
|
||||||
.. code:: toml
|
.. code:: toml
|
||||||
|
|
||||||
|
@ -46,13 +45,13 @@ from ipaddress import (
|
||||||
|
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
from searx.tools import config
|
|
||||||
from searx import settings
|
from searx import settings
|
||||||
|
|
||||||
from searx import redisdb
|
from searx import redisdb
|
||||||
from searx.redislib import incr_sliding_window, drop_counter
|
from searx.redislib import incr_sliding_window, drop_counter
|
||||||
|
|
||||||
from . import link_token
|
from . import link_token
|
||||||
|
from . import config
|
||||||
from ._helpers import (
|
from ._helpers import (
|
||||||
too_many_requests,
|
too_many_requests,
|
||||||
logger,
|
logger,
|
||||||
|
|
|
@ -33,7 +33,7 @@ from ipaddress import (
|
||||||
IPv6Address,
|
IPv6Address,
|
||||||
)
|
)
|
||||||
|
|
||||||
from searx.tools import config
|
from . import config
|
||||||
from ._helpers import logger
|
from ._helpers import logger
|
||||||
|
|
||||||
logger = logger.getChild('ip_limit')
|
logger = logger.getChild('ip_limit')
|
||||||
|
|
|
@ -99,15 +99,13 @@ def ping(request: flask.Request, token: str):
|
||||||
The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.
|
The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
from . import limiter # pylint: disable=import-outside-toplevel, cyclic-import
|
from . import redis_client, cfg # pylint: disable=import-outside-toplevel, cyclic-import
|
||||||
|
|
||||||
redis_client = redisdb.client()
|
|
||||||
if not redis_client:
|
if not redis_client:
|
||||||
return
|
return
|
||||||
if not token_is_valid(token):
|
if not token_is_valid(token):
|
||||||
return
|
return
|
||||||
|
|
||||||
cfg = limiter.get_cfg()
|
|
||||||
real_ip = ip_address(get_real_ip(request))
|
real_ip = ip_address(get_real_ip(request))
|
||||||
network = get_network(real_ip, cfg)
|
network = get_network(real_ip, cfg)
|
||||||
|
|
||||||
|
|
|
@ -1,15 +1,6 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
# lint: pylint
|
# lint: pylint
|
||||||
""".. _limiter src:
|
"""Bot protection / IP rate limitation. The intention of rate limitation is to
|
||||||
|
|
||||||
Limiter
|
|
||||||
=======
|
|
||||||
|
|
||||||
.. sidebar:: info
|
|
||||||
|
|
||||||
The limiter requires a :ref:`Redis <settings redis>` database.
|
|
||||||
|
|
||||||
Bot protection / IP rate limitation. The intention of rate limitation is to
|
|
||||||
limit suspicious requests from an IP. The motivation behind this is the fact
|
limit suspicious requests from an IP. The motivation behind this is the fact
|
||||||
that SearXNG passes through requests from bots and is thus classified as a bot
|
that SearXNG passes through requests from bots and is thus classified as a bot
|
||||||
itself. As a result, the SearXNG engine then receives a CAPTCHA or is blocked
|
itself. As a result, the SearXNG engine then receives a CAPTCHA or is blocked
|
||||||
|
@ -17,7 +8,40 @@ by the search engine (the origin) in some other way.
|
||||||
|
|
||||||
To avoid blocking, the requests from bots to SearXNG must also be blocked, this
|
To avoid blocking, the requests from bots to SearXNG must also be blocked, this
|
||||||
is the task of the limiter. To perform this task, the limiter uses the methods
|
is the task of the limiter. To perform this task, the limiter uses the methods
|
||||||
from the :py:obj:`searx.botdetection`.
|
from the :ref:`botdetection`:
|
||||||
|
|
||||||
|
- Analysis of the HTTP header in the request / :ref:`botdetection probe headers`
|
||||||
|
can be easily bypassed.
|
||||||
|
|
||||||
|
- Block and pass lists in which IPs are listed / :ref:`botdetection ip_lists`
|
||||||
|
are hard to maintain, since the IPs of bots are not all known and change over
|
||||||
|
the time.
|
||||||
|
|
||||||
|
- Detection & dynamically :ref:`botdetection rate limit` of bots based on the
|
||||||
|
behavior of the requests. For dynamically changeable IP lists a Redis
|
||||||
|
database is needed.
|
||||||
|
|
||||||
|
The prerequisite for IP based methods is the correct determination of the IP of
|
||||||
|
the client. The IP of the client is determined via the X-Forwarded-For_ HTTP
|
||||||
|
header.
|
||||||
|
|
||||||
|
.. attention::
|
||||||
|
|
||||||
|
A correct setup of the HTTP request headers ``X-Forwarded-For`` and
|
||||||
|
``X-Real-IP`` is essential to be able to assign a request to an IP correctly:
|
||||||
|
|
||||||
|
- `NGINX RequestHeader`_
|
||||||
|
- `Apache RequestHeader`_
|
||||||
|
|
||||||
|
.. _X-Forwarded-For:
|
||||||
|
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
|
||||||
|
.. _NGINX RequestHeader:
|
||||||
|
https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site
|
||||||
|
.. _Apache RequestHeader:
|
||||||
|
https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site
|
||||||
|
|
||||||
|
Enable Limiter
|
||||||
|
==============
|
||||||
|
|
||||||
To enable the limiter activate:
|
To enable the limiter activate:
|
||||||
|
|
||||||
|
@ -35,36 +59,72 @@ and set the redis-url connection. Check the value, it depends on your redis DB
|
||||||
redis:
|
redis:
|
||||||
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
|
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
|
||||||
|
|
||||||
|
|
||||||
|
Configure Limiter
|
||||||
|
=================
|
||||||
|
|
||||||
|
The methods of :ref:`botdetection` the limiter uses are configured in a local
|
||||||
|
file ``/etc/searxng/limiter.toml``. The defaults are shown in limiter.toml_ /
|
||||||
|
Don't copy all values to your local configuration, just enable what you need by
|
||||||
|
overwriting the defaults. For instance to activate the ``link_token`` method in
|
||||||
|
the :ref:`botdetection.ip_limit` you only need to set this option to ``true``:
|
||||||
|
|
||||||
|
.. code:: toml
|
||||||
|
|
||||||
|
[botdetection.ip_limit]
|
||||||
|
link_token = true
|
||||||
|
|
||||||
|
.. _limiter.toml:
|
||||||
|
|
||||||
|
``limiter.toml``
|
||||||
|
================
|
||||||
|
|
||||||
|
In this file the limiter finds the configuration of the :ref:`botdetection`:
|
||||||
|
|
||||||
|
- :ref:`botdetection ip_lists`
|
||||||
|
- :ref:`botdetection rate limit`
|
||||||
|
- :ref:`botdetection probe headers`
|
||||||
|
|
||||||
|
.. kernel-include:: $SOURCEDIR/limiter.toml
|
||||||
|
:code: toml
|
||||||
|
|
||||||
|
Implementation
|
||||||
|
==============
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
import sys
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from ipaddress import ip_address
|
from ipaddress import ip_address
|
||||||
import flask
|
import flask
|
||||||
import werkzeug
|
import werkzeug
|
||||||
|
|
||||||
from searx.tools import config
|
from searx import (
|
||||||
from searx import logger
|
logger,
|
||||||
|
redisdb,
|
||||||
from . import (
|
)
|
||||||
|
from searx import botdetection
|
||||||
|
from searx.botdetection import (
|
||||||
|
config,
|
||||||
http_accept,
|
http_accept,
|
||||||
http_accept_encoding,
|
http_accept_encoding,
|
||||||
http_accept_language,
|
http_accept_language,
|
||||||
http_user_agent,
|
http_user_agent,
|
||||||
ip_limit,
|
ip_limit,
|
||||||
ip_lists,
|
ip_lists,
|
||||||
)
|
|
||||||
|
|
||||||
from ._helpers import (
|
|
||||||
get_network,
|
get_network,
|
||||||
get_real_ip,
|
get_real_ip,
|
||||||
dump_request,
|
dump_request,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logger.getChild('botdetection.limiter')
|
# the configuration are limiter.toml and "limiter" in settings.yml so, for
|
||||||
|
# coherency, the logger is "limiter"
|
||||||
|
logger = logger.getChild('limiter')
|
||||||
|
|
||||||
CFG: config.Config = None # type: ignore
|
CFG: config.Config = None # type: ignore
|
||||||
|
_INSTALLED = False
|
||||||
|
|
||||||
LIMITER_CFG_SCHEMA = Path(__file__).parent / "limiter.toml"
|
LIMITER_CFG_SCHEMA = Path(__file__).parent / "limiter.toml"
|
||||||
"""Base configuration (schema) of the botdetection."""
|
"""Base configuration (schema) of the botdetection."""
|
||||||
|
@ -143,3 +203,31 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None:
|
||||||
return val
|
return val
|
||||||
logger.debug(f"OK {network}: %s", dump_request(flask.request))
|
logger.debug(f"OK {network}: %s", dump_request(flask.request))
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def pre_request():
|
||||||
|
"""See :py:obj:`flask.Flask.before_request`"""
|
||||||
|
return filter_request(flask.request)
|
||||||
|
|
||||||
|
|
||||||
|
def is_installed():
|
||||||
|
return _INSTALLED
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(app: flask.Flask, settings):
|
||||||
|
"""Instal the botlimiter aka limiter"""
|
||||||
|
global _INSTALLED # pylint: disable=global-statement
|
||||||
|
if not settings['server']['limiter'] and not settings['server']['public_instance']:
|
||||||
|
return
|
||||||
|
redis_client = redisdb.client()
|
||||||
|
if not redis_client:
|
||||||
|
logger.error(
|
||||||
|
"The limiter requires Redis, please consult the documentation: "
|
||||||
|
+ "https://docs.searxng.org/admin/searx.botdetection.html#limiter"
|
||||||
|
)
|
||||||
|
if settings['server']['public_instance']:
|
||||||
|
sys.exit(1)
|
||||||
|
return
|
||||||
|
botdetection.init(get_cfg(), redis_client)
|
||||||
|
app.before_request(pre_request)
|
||||||
|
_INSTALLED = True
|
|
@ -1,38 +0,0 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
# lint: pylint
|
|
||||||
# pyright: basic
|
|
||||||
"""see :ref:`limiter src`"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import flask
|
|
||||||
|
|
||||||
from searx import redisdb
|
|
||||||
from searx.plugins import logger
|
|
||||||
from searx.botdetection import limiter
|
|
||||||
|
|
||||||
name = "Request limiter"
|
|
||||||
description = "Limit the number of request"
|
|
||||||
default_on = False
|
|
||||||
preference_section = 'service'
|
|
||||||
|
|
||||||
logger = logger.getChild('limiter')
|
|
||||||
|
|
||||||
|
|
||||||
def pre_request():
|
|
||||||
"""See :ref:`flask.Flask.before_request`"""
|
|
||||||
return limiter.filter_request(flask.request)
|
|
||||||
|
|
||||||
|
|
||||||
def init(app: flask.Flask, settings) -> bool:
|
|
||||||
if not settings['server']['limiter'] and not settings['server']['public_instance']:
|
|
||||||
return False
|
|
||||||
if not redisdb.client():
|
|
||||||
logger.error(
|
|
||||||
"The limiter requires Redis, please consult the documentation: "
|
|
||||||
+ "https://docs.searxng.org/admin/searx.botdetection.html#limiter"
|
|
||||||
)
|
|
||||||
if settings['server']['public_instance']:
|
|
||||||
sys.exit(1)
|
|
||||||
return False
|
|
||||||
app.before_request(pre_request)
|
|
||||||
return True
|
|
|
@ -1,8 +0,0 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
# lint: pylint
|
|
||||||
""".. _tools src:
|
|
||||||
|
|
||||||
A collection of *utilities* used by SearXNG, but without SearXNG specific
|
|
||||||
peculiarities.
|
|
||||||
|
|
||||||
"""
|
|
|
@ -57,7 +57,9 @@ from searx import (
|
||||||
)
|
)
|
||||||
|
|
||||||
from searx import infopage
|
from searx import infopage
|
||||||
from searx.botdetection import limiter
|
from searx import limiter
|
||||||
|
from searx.botdetection import link_token
|
||||||
|
|
||||||
from searx.data import ENGINE_DESCRIPTIONS
|
from searx.data import ENGINE_DESCRIPTIONS
|
||||||
from searx.results import Timing
|
from searx.results import Timing
|
||||||
from searx.settings_defaults import OUTPUT_FORMATS
|
from searx.settings_defaults import OUTPUT_FORMATS
|
||||||
|
@ -94,7 +96,6 @@ from searx.utils import (
|
||||||
from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
|
from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
|
||||||
from searx.query import RawTextQuery
|
from searx.query import RawTextQuery
|
||||||
from searx.plugins import Plugin, plugins, initialize as plugin_initialize
|
from searx.plugins import Plugin, plugins, initialize as plugin_initialize
|
||||||
from searx.botdetection import link_token
|
|
||||||
from searx.plugins.oa_doi_rewrite import get_doi_resolver
|
from searx.plugins.oa_doi_rewrite import get_doi_resolver
|
||||||
from searx.preferences import (
|
from searx.preferences import (
|
||||||
Preferences,
|
Preferences,
|
||||||
|
@ -1288,7 +1289,7 @@ def config():
|
||||||
'DOCS_URL': get_setting('brand.docs_url'),
|
'DOCS_URL': get_setting('brand.docs_url'),
|
||||||
},
|
},
|
||||||
'limiter': {
|
'limiter': {
|
||||||
'enabled': settings['server']['limiter'],
|
'enabled': limiter.is_installed(),
|
||||||
'botdetection.ip_limit.link_token': _limiter_cfg.get('botdetection.ip_limit.link_token'),
|
'botdetection.ip_limit.link_token': _limiter_cfg.get('botdetection.ip_limit.link_token'),
|
||||||
'botdetection.ip_lists.pass_searxng_org': _limiter_cfg.get('botdetection.ip_lists.pass_searxng_org'),
|
'botdetection.ip_lists.pass_searxng_org': _limiter_cfg.get('botdetection.ip_lists.pass_searxng_org'),
|
||||||
},
|
},
|
||||||
|
@ -1322,6 +1323,7 @@ if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_
|
||||||
redis_initialize()
|
redis_initialize()
|
||||||
plugin_initialize(app)
|
plugin_initialize(app)
|
||||||
search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
|
search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
|
||||||
|
limiter.initialize(app, settings)
|
||||||
|
|
||||||
|
|
||||||
def run():
|
def run():
|
||||||
|
|
|
@ -1,6 +1,11 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
from searx import plugins
|
from searx import (
|
||||||
|
plugins,
|
||||||
|
limiter,
|
||||||
|
botdetection,
|
||||||
|
)
|
||||||
|
|
||||||
from mock import Mock
|
from mock import Mock
|
||||||
from tests import SearxTestCase
|
from tests import SearxTestCase
|
||||||
|
|
||||||
|
@ -46,6 +51,8 @@ class SelfIPTest(SearxTestCase):
|
||||||
plugin = plugins.load_and_initialize_plugin('searx.plugins.self_info', False, (None, {}))
|
plugin = plugins.load_and_initialize_plugin('searx.plugins.self_info', False, (None, {}))
|
||||||
store = plugins.PluginStore()
|
store = plugins.PluginStore()
|
||||||
store.register(plugin)
|
store.register(plugin)
|
||||||
|
cfg = limiter.get_cfg()
|
||||||
|
botdetection.init(cfg, None)
|
||||||
|
|
||||||
self.assertTrue(len(store.plugins) == 1)
|
self.assertTrue(len(store.plugins) == 1)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user