From 90de8e22a04cd68e9f7c37e96c59c2514266141e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 15 Feb 2021 13:30:28 -0500 Subject: [PATCH] API keys support, bug fixes, improvements --- .gitignore | 2 + README.md | 31 +++++++++++++- app/__init__.py | 1 + app/api_keys.py | 54 ++++++++++++++++++++++++ app/app.py | 91 ++++++++++++++++++++++++++++++---------- app/main.py | 15 +++---- app/manage.py | 45 ++++++++++++++++++++ app/templates/index.html | 39 +++++++++++++---- manage.py | 4 ++ requirements.txt | 1 + setup.py | 3 +- 11 files changed, 242 insertions(+), 44 deletions(-) create mode 100644 app/api_keys.py create mode 100644 app/manage.py create mode 100644 manage.py diff --git a/.gitignore b/.gitignore index bb44dea..256d239 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,5 @@ dmypy.json .pyre/ installed_models/ +# Misc +api_keys.db diff --git a/README.md b/README.md index 20f3d05..beefe43 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,34 @@ docker-compose up -d --build | --frontend-language-source | Set frontend default language - source | `en` | | --frontend-language-target | Set frontend default language - target | `es` | | --frontend-timeout | Set frontend translation timeout | `500` | +| --offline | Run user-interface entirely offline (don't use internet CDNs) | `false` | +| --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` | +## Manage API Keys + +LibreTranslate supports per-user limit quotas, e.g. you can issue API keys to users so that they can enjoy higher requests limits per minute (if you also set `--req-limit`). By default all users are rate-limited based on `--req-limit`, but passing an optional `api_key` parameter to the REST endpoints allows a user to enjoy higher request limits. + +To use API keys simply start LibreTranslate with the `--api-keys` option. + +### Add New Keys + +To issue a new API key with 120 requests per minute limits: + +```bash +ltmanage keys add 120 +``` + +### Remove Keys + +```bash +ltmanage keys remove +``` + +### View Keys + +```bash +ltmanage keys +``` ## Roadmap @@ -120,14 +147,14 @@ Help us by opening a pull request! - [x] A docker image (thanks [@vemonet](https://github.com/vemonet) !) - [x] Auto-detect input language (thanks [@vemonet](https://github.com/vemonet) !) -- [ ] User authentication / tokens +- [X] User authentication / tokens - [ ] Language bindings for every computer language ## FAQ ### Can I use your API server at libretranslate.com for my application in production? -The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to discuss options. +The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to get an API key or discuss other options. ## Credits diff --git a/app/__init__.py b/app/__init__.py index c28a133..1fb8330 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1,2 @@ from .main import main +from .manage import manage diff --git a/app/api_keys.py b/app/api_keys.py new file mode 100644 index 0000000..14c6518 --- /dev/null +++ b/app/api_keys.py @@ -0,0 +1,54 @@ +import sqlite3 +import uuid +from expiringdict import ExpiringDict + +DEFAULT_DB_PATH = "api_keys.db" + +class Database: + def __init__(self, db_path = DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): + self.db_path = db_path + self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) + + # Make sure to do data synchronization on writes! + self.c = sqlite3.connect(db_path, check_same_thread=False) + self.c.execute('''CREATE TABLE IF NOT EXISTS api_keys ( + "api_key" TEXT NOT NULL, + "req_limit" INTEGER NOT NULL, + PRIMARY KEY("api_key") + );''') + + def lookup(self, api_key): + req_limit = self.cache.get(api_key) + if req_limit is None: + # DB Lookup + stmt = self.c.execute('SELECT req_limit FROM api_keys WHERE api_key = ?', (api_key, )) + row = stmt.fetchone() + if row is not None: + self.cache[api_key] = row[0] + req_limit = row[0] + else: + self.cache[api_key] = False + req_limit = False + + if isinstance(req_limit, bool): + req_limit = None + + return req_limit + + def add(self, req_limit, api_key = "auto"): + if api_key == "auto": + api_key = str(uuid.uuid4()) + + self.remove(api_key) + self.c.execute("INSERT INTO api_keys (api_key, req_limit) VALUES (?, ?)", (api_key, req_limit)) + self.c.commit() + return (api_key, req_limit) + + def remove(self, api_key): + self.c.execute('DELETE FROM api_keys WHERE api_key = ?', (api_key, )) + self.c.commit() + return api_key + + def all(self): + row = self.c.execute("SELECT api_key, req_limit FROM api_keys") + return row.fetchall() \ No newline at end of file diff --git a/app/app.py b/app/app.py index c779193..84c0a7e 100644 --- a/app/app.py +++ b/app/app.py @@ -1,11 +1,16 @@ +import os from flask import Flask, render_template, jsonify, request, abort, send_from_directory from flask_swagger import swagger from flask_swagger_ui import get_swaggerui_blueprint from langdetect import detect_langs from langdetect import DetectorFactory from pkg_resources import resource_filename +from .api_keys import Database + DetectorFactory.seed = 0 # deterministic +api_keys_db = None + def get_remote_address(): if request.headers.getlist("X-Forwarded-For"): ip = request.headers.getlist("X-Forwarded-For")[0] @@ -14,8 +19,32 @@ def get_remote_address(): return ip -def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=False, frontend_language_source="en", frontend_language_target="en", frontend_timeout=500, offline=False): - if not offline: +def get_routes_limits(default_req_limit, api_keys_db): + if default_req_limit == -1: + # TODO: better way? + default_req_limit = 9999999999999 + + def limits(): + req_limit = default_req_limit + + if api_keys_db: + if request.is_json: + json = request.get_json() + api_key = json.get('api_key') + else: + api_key = request.values.get("api_key") + + if api_key: + db_req_limit = api_keys_db.lookup(api_key) + if db_req_limit is not None: + req_limit = db_req_limit + + return "%s per minute" % req_limit + + return [limits] + +def create_app(args): + if not args.offline: from app.init import boot boot() @@ -27,32 +56,32 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa for l in languages: language_map[l.code] = l.name - if debug: + if args.debug: app.config['TEMPLATES_AUTO_RELOAD'] = True # Map userdefined frontend languages to argos language object. - if frontend_language_source == "auto": + if args.frontend_language_source == "auto": frontend_argos_language_source = type('obj', (object,), { 'code': 'auto', 'name': 'Auto Detect' }) else: - frontend_argos_language_source = next(iter([l for l in languages if l.code == frontend_language_source]), None) + frontend_argos_language_source = next(iter([l for l in languages if l.code == args.frontend_language_source]), None) - frontend_argos_language_target = next(iter([l for l in languages if l.code == frontend_language_target]), None) + frontend_argos_language_target = next(iter([l for l in languages if l.code == args.frontend_language_target]), None) # Raise AttributeError to prevent app startup if user input is not valid. if frontend_argos_language_source is None: - raise AttributeError(f"{frontend_language_source} as frontend source language is not supported.") + raise AttributeError(f"{args.frontend_language_source} as frontend source language is not supported.") if frontend_argos_language_target is None: - raise AttributeError(f"{frontend_language_target} as frontend target language is not supported.") + raise AttributeError(f"{args.frontend_language_target} as frontend target language is not supported.") - if req_limit > 0: + if args.req_limit > 0 or args.api_keys: from flask_limiter import Limiter limiter = Limiter( app, key_func=get_remote_address, - default_limits=["%s per minute" % req_limit] + default_limits=get_routes_limits(args.req_limit, Database() if args.api_keys else None) ) @app.errorhandler(400) @@ -68,10 +97,12 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa return jsonify({"error": "Slowdown: " + str(e.description)}), 429 @app.route("/") + @limiter.exempt def index(): - return render_template('index.html', gaId=ga_id, frontendTimeout=frontend_timeout, offline=offline) + return render_template('index.html', gaId=args.ga_id, frontendTimeout=args.frontend_timeout, offline=args.offline, api_keys=args.api_keys, web_version=os.environ.get('LT_WEB') is not None) @app.route("/languages", methods=['GET', 'POST']) + @limiter.exempt def langs(): """ Retrieve list of supported languages @@ -149,6 +180,13 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa example: es required: true description: Target language code + - in: formData + name: api_key + schema: + type: string + example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + required: false + description: API key responses: 200: description: Translated text @@ -209,19 +247,19 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa batch = isinstance(q, list) - if batch and batch_limit != -1: + if batch and args.batch_limit != -1: batch_size = len(q) - if batch_limit < batch_size: - abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, batch_limit)) + if args.batch_limit < batch_size: + abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, args.batch_limit)) - if char_limit != -1: + if args.char_limit != -1: if batch: chars = sum([len(text) for text in q]) else: chars = len(q) - if char_limit < chars: - abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, char_limit)) + if args.char_limit < chars: + abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, args.char_limit)) if source_lang == 'auto': candidate_langs = list(filter(lambda l: l.lang in language_map, detect_langs(q))) @@ -229,7 +267,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa if len(candidate_langs) > 0: candidate_langs.sort(key=lambda l: l.prob, reverse=True) - if debug: + if args.debug: print(candidate_langs) source_lang = next(iter([l.code for l in languages if l.code == candidate_langs[0].lang]), None) @@ -238,7 +276,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa else: source_lang = 'en' - if debug: + if args.debug: print("Auto detected: %s" % source_lang) src_lang = next(iter([l for l in languages if l.code == source_lang]), None) @@ -274,6 +312,13 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa example: Hello world! required: true description: Text to detect + - in: formData + name: api_key + schema: + type: string + example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + required: false + description: API key responses: 200: description: Detections @@ -340,6 +385,7 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa @app.route("/frontend/settings") + @limiter.exempt def frontend_settings(): """ Retrieve frontend specific settings @@ -381,18 +427,19 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa type: string description: Human-readable language name (in English) """ - return jsonify({'charLimit': char_limit, - 'frontendTimeout': frontend_timeout, + return jsonify({'charLimit': args.char_limit, + 'frontendTimeout': args.frontend_timeout, 'language': { 'source': {'code': frontend_argos_language_source.code, 'name': frontend_argos_language_source.name}, 'target': {'code': frontend_argos_language_target.code, 'name': frontend_argos_language_target.name}} }) swag = swagger(app) - swag['info']['version'] = "1.0" + swag['info']['version'] = "1.2" swag['info']['title'] = "LibreTranslate" @app.route("/spec") + @limiter.exempt def spec(): return jsonify(swag) diff --git a/app/main.py b/app/main.py index bae05bb..ffc3ef2 100644 --- a/app/main.py +++ b/app/main.py @@ -10,7 +10,7 @@ def main(): parser.add_argument('--char-limit', default=-1, type=int, metavar="", help='Set character limit (%(default)s)') parser.add_argument('--req-limit', default=-1, type=int, metavar="", - help='Set maximum number of requests per minute per client (%(default)s)') + help='Set the default maximum number of requests per minute per client (%(default)s)') parser.add_argument('--batch-limit', default=-1, type=int, metavar="", help='Set maximum number of texts to translate in a batch request (%(default)s)') parser.add_argument('--ga-id', type=str, default=None, metavar="", @@ -27,18 +27,13 @@ def main(): help='Set frontend translation timeout (%(default)s)') parser.add_argument('--offline', default=False, action="store_true", help="Use offline") + parser.add_argument('--api-keys', default=False, action="store_true", + help="Enable API keys database for per-user rate limits lookup") + args = parser.parse_args() + app = create_app(args) - app = create_app(char_limit=args.char_limit, - req_limit=args.req_limit, - batch_limit=args.batch_limit, - ga_id=args.ga_id, - debug=args.debug, - frontend_language_source=args.frontend_language_source, - frontend_language_target=args.frontend_language_target, - frontend_timeout=args.frontend_timeout, - offline=args.offline) if args.debug: app.run(host=args.host, port=args.port) else: diff --git a/app/manage.py b/app/manage.py new file mode 100644 index 0000000..103a61b --- /dev/null +++ b/app/manage.py @@ -0,0 +1,45 @@ +import argparse +from app.api_keys import Database + +def manage(): + parser = argparse.ArgumentParser(description='LibreTranslate Manage Tools') + subparsers = parser.add_subparsers(help='', dest='command', required=True, title="Command List") + + keys_parser = subparsers.add_parser('keys', help='Manage API keys database') + keys_subparser = keys_parser.add_subparsers(help='', dest='sub_command', title="Command List") + + keys_add_parser = keys_subparser.add_parser('add', help='Add API keys to database') + keys_add_parser.add_argument('req_limit', + type=int, + help='Request Limits (per second)') + keys_add_parser.add_argument('--key', + type=str, + default="auto", + required=False, + help='API Key') + + keys_remove_parser = keys_subparser.add_parser('remove', help='Remove API keys to database') + keys_remove_parser.add_argument('key', + type=str, + help='API Key') + + args = parser.parse_args() + + if args.command == 'keys': + db = Database() + if args.sub_command is None: + # Print keys + keys = db.all() + if not keys: + print("There are no API keys") + else: + for item in keys: + print("%s: %s" % item) + + elif args.sub_command == 'add': + print(db.add(args.req_limit, args.key)[0]) + elif args.sub_command == 'remove': + print(db.remove(args.key)) + else: + parser.print_help() + exit(1) \ No newline at end of file diff --git a/app/templates/index.html b/app/templates/index.html index 82c2fe7..eaedc1f 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -60,13 +60,19 @@
+
  • GitHub
    • + {% if api_keys %} +
  • vpn_key
    • + {% endif %} + menu
    @@ -131,7 +137,7 @@
    @@ -197,7 +203,7 @@ - + {% if web_version %}
    @@ -210,20 +216,25 @@
    - +{% endif %}