From 1ac27aeb8270158403a644d5b1c685218bbbff6e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 16 May 2021 11:50:22 -0400 Subject: [PATCH] Flood protection --- app/app.py | 37 ++++++++++++++++++++++++++++++++++--- app/flood.py | 36 ++++++++++++++++++++++++++++++++++++ app/main.py | 2 ++ requirements.txt | 1 + 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 app/flood.py diff --git a/app/app.py b/app/app.py index d0723e1..e0af8de 100644 --- a/app/app.py +++ b/app/app.py @@ -5,8 +5,7 @@ from flask_swagger_ui import get_swaggerui_blueprint from pkg_resources import resource_filename from .api_keys import Database from app.language import detect_languages, transliterate - -api_keys_db = None +from app import flood def get_json_dict(request): d = request.get_json() @@ -88,6 +87,9 @@ def create_app(args): else: from .no_limiter import Limiter limiter = Limiter() + + if args.req_flood_threshold > 0: + flood.setup(args.req_flood_threshold) @app.errorhandler(400) def invalid_api(e): @@ -99,8 +101,14 @@ def create_app(args): @app.errorhandler(429) def slow_down_error(e): + flood.report(get_remote_address()) return jsonify({"error": "Slowdown: " + str(e.description)}), 429 + @app.errorhandler(403) + def denied(e): + return jsonify({"error": str(e.description)}), 403 + + @app.route("/") @limiter.exempt def index(): @@ -236,7 +244,18 @@ def create_app(args): error: type: string description: Reason for slow down + 403: + description: Banned + schema: + id: error-response + type: object + properties: + error: + type: string + description: Error message """ + if flood.is_banned(get_remote_address()): + abort(403, description="Too many request limits violations") if request.is_json: json = get_json_dict(request) @@ -350,7 +369,7 @@ def create_app(args): properties: error: type: string - description: Error message + description: Error message 500: description: Detection error schema: @@ -369,7 +388,19 @@ def create_app(args): error: type: string description: Reason for slow down + 403: + description: Banned + schema: + id: error-response + type: object + properties: + error: + type: string + description: Error message """ + if flood.is_banned(get_remote_address()): + abort(403, description="Too many request limits violations") + if request.is_json: json = get_json_dict(request) q = json.get('q') diff --git a/app/flood.py b/app/flood.py new file mode 100644 index 0000000..377d85e --- /dev/null +++ b/app/flood.py @@ -0,0 +1,36 @@ +import time +import atexit + +from apscheduler.schedulers.background import BackgroundScheduler + +banned = {} +active = False +threshold = -1 + +def clear_banned(): + global banned + print(banned) + banned = {} + +def setup(violations_threshold = 100): + global active + global threshold + + active = True + threshold = violations_threshold + + scheduler = BackgroundScheduler() + scheduler.add_job(func=clear_banned, trigger="interval", weeks=4) + scheduler.start() + + # Shut down the scheduler when exiting the app + atexit.register(lambda: scheduler.shutdown()) + + +def report(request_ip): + banned[request_ip] = banned.get(request_ip, 0) + banned[request_ip] += 1 + +def is_banned(request_ip): + # More than X offences? + return active and banned.get(request_ip, 0) >= threshold \ No newline at end of file diff --git a/app/main.py b/app/main.py index 55a7d62..ce1d5f9 100644 --- a/app/main.py +++ b/app/main.py @@ -14,6 +14,8 @@ def main(): help='Set the default maximum number of requests per minute per client (%(default)s)') parser.add_argument('--daily-req-limit', default=-1, type=int, metavar="", help='Set the default maximum number of requests per day per client, in addition to req-limit. (%(default)s)') + parser.add_argument('--req-flood-threshold', default=-1, type=int, metavar="", + help='Set the maximum number of request limit offences per 4 weeks that a client can exceed before being banned. (%(default)s)') parser.add_argument('--batch-limit', default=-1, type=int, metavar="", help='Set maximum number of texts to translate in a batch request (%(default)s)') parser.add_argument('--ga-id', type=str, default=None, metavar="", diff --git a/requirements.txt b/requirements.txt index e13d5d9..f44a9c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ pycld2==0.41 morfessor==2.0.6 polyglot==16.7.4 appdirs==1.4.4 +APScheduler==3.7.0