mirror of
https://github.com/searxng/searxng.git
synced 2024-11-04 20:30:11 +01:00
[mod] plugins/limiter.py - use searx.redislib.incr_sliding_window
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
parent
08876df7c2
commit
4b185f0e11
@ -13,11 +13,11 @@ Enable the plugin in ``settings.yml``:
|
||||
- ``redis.url: ...`` check the value, see :ref:`settings redis`
|
||||
"""
|
||||
|
||||
import hmac
|
||||
import re
|
||||
from flask import request
|
||||
|
||||
from searx.shared import redisdb
|
||||
from searx.redislib import incr_sliding_window
|
||||
|
||||
name = "Request limiter"
|
||||
description = "Limit the number of request"
|
||||
@ -36,8 +36,9 @@ re_bot = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def is_accepted_request(inc_get_counter) -> bool:
|
||||
def is_accepted_request() -> bool:
|
||||
# pylint: disable=too-many-return-statements
|
||||
redis_client = redisdb.client()
|
||||
user_agent = request.headers.get('User-Agent', '')
|
||||
x_forwarded_for = request.headers.get('X-Forwarded-For', '')
|
||||
|
||||
@ -47,70 +48,45 @@ def is_accepted_request(inc_get_counter) -> bool:
|
||||
return True
|
||||
|
||||
if request.path == '/search':
|
||||
c_burst = inc_get_counter(interval=20, keys=[b'IP limit, burst', x_forwarded_for])
|
||||
c_10min = inc_get_counter(interval=600, keys=[b'IP limit, 10 minutes', x_forwarded_for])
|
||||
c_burst = incr_sliding_window(redis_client, 'IP limit, burst' + x_forwarded_for, 20)
|
||||
c_10min = incr_sliding_window(redis_client, 'IP limit, 10 minutes' + x_forwarded_for, 600)
|
||||
if c_burst > 15 or c_10min > 150:
|
||||
logger.debug("to many request") # pylint: disable=undefined-variable
|
||||
return False
|
||||
|
||||
if re_bot.match(user_agent):
|
||||
logger.debug("detected bot") # pylint: disable=undefined-variable
|
||||
return False
|
||||
|
||||
if len(request.headers.get('Accept-Language', '').strip()) == '':
|
||||
logger.debug("missing Accept-Language") # pylint: disable=undefined-variable
|
||||
return False
|
||||
|
||||
if request.headers.get('Connection') == 'close':
|
||||
logger.debug("got Connection=close") # pylint: disable=undefined-variable
|
||||
return False
|
||||
|
||||
accept_encoding_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')]
|
||||
if 'gzip' not in accept_encoding_list or 'deflate' not in accept_encoding_list:
|
||||
logger.debug("suspicious Accept-Encoding") # pylint: disable=undefined-variable
|
||||
return False
|
||||
|
||||
if 'text/html' not in request.accept_mimetypes:
|
||||
logger.debug("Accept-Encoding misses text/html") # pylint: disable=undefined-variable
|
||||
return False
|
||||
|
||||
if request.args.get('format', 'html') != 'html':
|
||||
c = inc_get_counter(interval=3600, keys=[b'API limit', x_forwarded_for])
|
||||
c = incr_sliding_window(redis_client, 'API limit' + x_forwarded_for, 3600)
|
||||
if c > 4:
|
||||
logger.debug("API limit exceeded") # pylint: disable=undefined-variable
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def create_inc_get_counter(redis_client, secret_key_bytes):
|
||||
lua_script = """
|
||||
local slidingWindow = KEYS[1]
|
||||
local key = KEYS[2]
|
||||
local now = tonumber(redis.call('TIME')[1])
|
||||
local id = redis.call('INCR', 'counter')
|
||||
if (id > 2^46)
|
||||
then
|
||||
redis.call('SET', 'count', 0)
|
||||
end
|
||||
redis.call('ZREMRANGEBYSCORE', key, 0, now - slidingWindow)
|
||||
redis.call('ZADD', key, now, id)
|
||||
local result = redis.call('ZCOUNT', key, 0, now+1)
|
||||
redis.call('EXPIRE', key, slidingWindow)
|
||||
return result
|
||||
"""
|
||||
script_sha = redis_client.script_load(lua_script)
|
||||
|
||||
def inc_get_counter(interval, keys):
|
||||
m = hmac.new(secret_key_bytes, digestmod='sha256')
|
||||
for k in keys:
|
||||
m.update(bytes(str(k), encoding='utf-8') or b'')
|
||||
m.update(b"\0")
|
||||
key = m.digest()
|
||||
return redis_client.evalsha(script_sha, 2, interval, key)
|
||||
|
||||
return inc_get_counter
|
||||
|
||||
|
||||
def create_pre_request(get_aggregation_count):
|
||||
def pre_request():
|
||||
if not is_accepted_request(get_aggregation_count):
|
||||
return '', 429
|
||||
return None
|
||||
|
||||
return pre_request
|
||||
def pre_request():
|
||||
if not is_accepted_request():
|
||||
return '', 429
|
||||
return None
|
||||
|
||||
|
||||
def init(app, settings):
|
||||
@ -122,8 +98,5 @@ def init(app, settings):
|
||||
logger.error("init limiter DB failed!!!") # pylint: disable=undefined-variable
|
||||
return False
|
||||
|
||||
redis_client = redisdb.client()
|
||||
secret_key_bytes = bytes(settings['server']['secret_key'], encoding='utf-8')
|
||||
inc_get_counter = create_inc_get_counter(redis_client, secret_key_bytes)
|
||||
app.before_request(create_pre_request(inc_get_counter))
|
||||
app.before_request(pre_request)
|
||||
return True
|
||||
|
Loading…
Reference in New Issue
Block a user