mirror of
https://github.com/searxng/searxng.git
synced 2024-11-04 20:30:11 +01:00
[mod] limiter: ip_limt - monitore suspicious IPs
To intercept bots that get their IPs from a range of IPs, there is a ``SUSPICIOUS_IP_WINDOW``. In this window the suspicious IPs are stored for a longer time. IPs stored in this sliding window have a maximum of ``SUSPICIOUS_IP_MAX`` accesses before they are blocked. As soon as the IP makes a request that is not suspicious, the sliding window for this IP is droped. Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
parent
9d7456fd6c
commit
52f1452c09
@ -25,6 +25,13 @@ the request rates are reduced:
|
||||
- :py:obj:`BURST_MAX` -> :py:obj:`BURST_MAX_SUSPICIOUS`
|
||||
- :py:obj:`LONG_MAX` -> :py:obj:`LONG_MAX_SUSPICIOUS`
|
||||
|
||||
To intercept bots that get their IPs from a range of IPs, there is a
|
||||
:py:obj:`SUSPICIOUS_IP_WINDOW`. In this window the suspicious IPs are stored
|
||||
for a longer time. IPs stored in this sliding window have a maximum of
|
||||
:py:obj:`SUSPICIOUS_IP_MAX` accesses before they are blocked. As soon as the IP
|
||||
makes a request that is not suspicious, the sliding window for this IP is
|
||||
droped.
|
||||
|
||||
.. _X-Forwarded-For:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
|
||||
|
||||
@ -37,7 +44,7 @@ from searx.tools import config
|
||||
|
||||
from searx import redisdb
|
||||
from searx import logger
|
||||
from searx.redislib import incr_sliding_window
|
||||
from searx.redislib import incr_sliding_window, drop_counter
|
||||
|
||||
from . import link_token
|
||||
|
||||
@ -67,6 +74,12 @@ API_WONDOW = 3600
|
||||
API_MAX = 4
|
||||
"""Maximum requests from one IP in the :py:obj:`API_WONDOW`"""
|
||||
|
||||
SUSPICIOUS_IP_WINDOW = 3600 * 24
|
||||
"""Time (sec) before sliding window for one suspicious IP expires."""
|
||||
|
||||
SUSPICIOUS_IP_MAX = 3
|
||||
"""Maximum requests from one suspicious IP in the :py:obj:`SUSPICIOUS_IP_WINDOW`."""
|
||||
|
||||
|
||||
def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple[int, str]]:
|
||||
redis_client = redisdb.client()
|
||||
@ -81,10 +94,18 @@ def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple
|
||||
return 429, "BLOCK %s: API limit exceeded"
|
||||
|
||||
suspicious = False
|
||||
suspicious_ip_counter = 'IP limit - SUSPICIOUS_IP_WINDOW:' + x_forwarded_for
|
||||
|
||||
if cfg['botdetection.ip_limit.link_token']:
|
||||
suspicious = link_token.is_suspicious(request)
|
||||
|
||||
if suspicious:
|
||||
|
||||
# this IP is suspicious: count requests from this IP
|
||||
c = incr_sliding_window(redis_client, suspicious_ip_counter, SUSPICIOUS_IP_WINDOW)
|
||||
if c > SUSPICIOUS_IP_MAX:
|
||||
return 429, f"bot detected, too many request from {x_forwarded_for} in SUSPICIOUS_IP_WINDOW"
|
||||
|
||||
c = incr_sliding_window(redis_client, 'IP limit - BURST_WINDOW:' + x_forwarded_for, BURST_WINDOW)
|
||||
if c > BURST_MAX_SUSPICIOUS:
|
||||
return 429, f"bot detected, too many request from {x_forwarded_for} in BURST_MAX_SUSPICIOUS"
|
||||
@ -94,6 +115,11 @@ def filter_request(request: flask.Request, cfg: config.Config) -> Optional[Tuple
|
||||
return 429, f"bot detected, too many request from {x_forwarded_for} in LONG_MAX_SUSPICIOUS"
|
||||
|
||||
else:
|
||||
|
||||
if cfg['botdetection.ip_limit.link_token']:
|
||||
# this IP is no longer suspicious: release ip again / delete the counter of this IP
|
||||
drop_counter(redis_client, suspicious_ip_counter)
|
||||
|
||||
c = incr_sliding_window(redis_client, 'IP limit - BURST_WINDOW:' + x_forwarded_for, BURST_WINDOW)
|
||||
if c > BURST_MAX:
|
||||
return 429, f"bot detected, too many request from {x_forwarded_for} in BURST_MAX"
|
||||
|
Loading…
Reference in New Issue
Block a user