diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index d2404cf..a2b5ebb 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index b183d08..a68ae51 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -9,35 +9,8 @@ on: jobs: tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest flake8 - pip install . - python scripts/compile_locales.py - - - name: Check code style with flake8 (lint) - run: | - # warnings if there are Python syntax errors or undefined names - # (remove --exit-zero to fail when syntax error) - flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Test with pytest - run: pytest + uses: LibreTranslate/LibreTranslate/.github/workflows/run-tests.yml@main + secrets: inherit publish: @@ -45,23 +18,23 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - python setup.py sdist bdist_wheel + pip install build - - name: Build and publish to PyPI - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - name: Build run: | pip install Babel==2.11.0 python scripts/compile_locales.py - python setup.py sdist bdist_wheel - twine upload dist/* + python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: ${{ secrets.PYPI_USERNAME }} + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 9e59bfd..2703870 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -2,6 +2,7 @@ name: Run tests # Run test at each push to main, if changes to package or tests files on: workflow_dispatch: + workflow_call: pull_request: branches: [ main ] push: @@ -21,35 +22,26 @@ jobs: python-version: ['3.8', '3.9', '3.10'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pytest flake8 - pip install . - - - name: Check code style with flake8 (lint) - run: | - # warnings if there are Python syntax errors or undefined names - # (remove --exit-zero to fail when syntax error) - flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + pipx install hatch + hatch run locales - name: Test with pytest - run: pytest -v + run: hatch run test test_docker_build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Docker build run: docker build -f docker/Dockerfile -t libretranslate . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9621360..d5add11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,17 +20,57 @@ sudo dnf install cmake ## Getting Started +Install [`hatch`](https://hatch.pypa.io) to manage the projects dependencies and run dev scripts: + +```bash +pipx install hatch +``` + +Clone the repository: + ```bash git clone https://github.com/LibreTranslate/LibreTranslate.git cd LibreTranslate -pip install -e . -libretranslate [args] +``` +Run in development: + +```bash +hatch run dev +``` + +Then open a web browser to + +You can also start a new shell in a virtual environment with libretranslate installed: + +```bash +hatch shell +libretranslate [args] # Or python main.py [args] ``` -Then open a web browser to +> You can still use `pip install -e ".[test]"` directly if you don't want to use hatch. + +## Run the tests + +Run the test suite and linting checks: + +```bash +hatch run test +``` + +To display all `print()` when debugging: + +```bash +hatch run test -s +``` + +You can also run the tests on multiple python versions: + +```bash +hatch run all:test +``` ## Run with Docker diff --git a/libretranslate/api_keys.py b/libretranslate/api_keys.py index dc1da3e..ea9e603 100644 --- a/libretranslate/api_keys.py +++ b/libretranslate/api_keys.py @@ -1,8 +1,10 @@ import os import sqlite3 import uuid + import requests from expiringdict import ExpiringDict + from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH'] @@ -12,14 +14,14 @@ class Database: def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): # Legacy check - this can be removed at some point in the near future if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"): - print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db")) + print("Migrating {} to {}".format("api_keys.db", "db/api_keys.db")) try: os.rename("api_keys.db", "db/api_keys.db") except Exception as e: print(str(e)) db_dir = os.path.dirname(db_path) - if not db_dir == "" and not os.path.exists(db_dir): + if db_dir != '' and not os.path.exists(db_dir): os.makedirs(db_dir) self.db_path = db_path self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) @@ -85,16 +87,13 @@ class RemoteDatabase: req_limit = self.cache.get(api_key) if req_limit is None: try: - r = requests.post(self.url, data={'api_key': api_key}) + r = requests.post(self.url, data={'api_key': api_key}, timeout=60) res = r.json() except Exception as e: print("Cannot authenticate API key: " + str(e)) return None - if res.get('error', None) is None: - req_limit = res.get('req_limit', None) - else: - req_limit = None + req_limit = res.get('req_limit', None) if res.get('error', None) is None else None self.cache[api_key] = req_limit return req_limit diff --git a/libretranslate/app.py b/libretranslate/app.py index c5fe485..39ad1ec 100644 --- a/libretranslate/app.py +++ b/libretranslate/app.py @@ -1,30 +1,37 @@ import io import os -import tempfile import re +import tempfile import uuid +from datetime import datetime from functools import wraps from html import unescape from timeit import default_timer -from datetime import datetime import argostranslatefiles from argostranslatefiles import get_supported_formats -from flask import (abort, Blueprint, Flask, jsonify, render_template, request, - Response, send_file, url_for, session) +from flask import Blueprint, Flask, Response, abort, jsonify, render_template, request, send_file, session, url_for +from flask_babel import Babel +from flask_session import Session from flask_swagger import swagger from flask_swagger_ui import get_swaggerui_blueprint -from flask_session import Session from translatehtml import translate_html -from werkzeug.utils import secure_filename from werkzeug.exceptions import HTTPException from werkzeug.http import http_date -from flask_babel import Babel +from werkzeug.utils import secure_filename -from libretranslate import scheduler, flood, secret, remove_translated_files, security, storage +from libretranslate import flood, remove_translated_files, scheduler, secret, security, storage from libretranslate.language import detect_languages, improve_translation_formatting -from libretranslate.locales import (_, _lazy, get_available_locales, get_available_locale_codes, gettext_escaped, - gettext_html, lazy_swag, get_alternate_locale_links) +from libretranslate.locales import ( + _, + _lazy, + get_alternate_locale_links, + get_available_locale_codes, + get_available_locales, + gettext_escaped, + gettext_html, + lazy_swag, +) from .api_keys import Database, RemoteDatabase from .suggestions import Database as SuggestionsDatabase @@ -122,8 +129,8 @@ def create_app(args): from libretranslate.language import load_languages - SWAGGER_URL = args.url_prefix + "/docs" # Swagger UI (w/o trailing '/') - API_URL = args.url_prefix + "/spec" + swagger_url = args.url_prefix + "/docs" # Swagger UI (w/o trailing '/') + api_url = args.url_prefix + "/spec" bp = Blueprint('Main app', __name__) @@ -150,10 +157,7 @@ def create_app(args): frontend_argos_language_source = languages[0] - if len(languages) >= 2: - language_target_fallback = languages[1] - else: - language_target_fallback = languages[0] + language_target_fallback = languages[1] if len(languages) >= 2 else languages[0] if args.frontend_language_target == "locale": def resolve_language_locale(): @@ -185,10 +189,7 @@ def create_app(args): if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0: api_keys_db = None if args.api_keys: - if args.api_keys_remote: - api_keys_db = RemoteDatabase(args.api_keys_remote) - else: - api_keys_db = Database(args.api_keys_db_path) + api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path) from flask_limiter import Limiter @@ -220,7 +221,7 @@ def create_app(args): os.mkdir(default_mp_dir) os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir - from prometheus_client import CONTENT_TYPE_LATEST, Summary, Gauge, CollectorRegistry, multiprocess, generate_latest + from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Gauge, Summary, generate_latest, multiprocess @bp.route("/metrics") @limiter.exempt @@ -338,7 +339,7 @@ def create_app(args): get_api_key_link=args.get_api_key_link, web_version=os.environ.get("LT_WEB") is not None, version=get_version(), - swagger_url=SWAGGER_URL, + swagger_url=swagger_url, available_locales=[{'code': l['code'], 'name': _lazy(l['name'])} for l in get_available_locales(not args.debug)], current_locale=get_locale(), alternate_locales=get_alternate_locale_links() @@ -544,10 +545,7 @@ def create_app(args): ) if args.char_limit != -1: - if batch: - chars = sum([len(text) for text in q]) - else: - chars = len(q) + chars = sum([len(text) for text in q]) if batch else len(q) if args.char_limit < chars: abort( @@ -557,10 +555,7 @@ def create_app(args): if source_lang == "auto": source_langs = [] - if batch: - auto_detect_texts = q - else: - auto_detect_texts = [q] + auto_detect_texts = q if batch else [q] overall_candidates = detect_languages(q) @@ -797,7 +792,7 @@ def create_app(args): checked_filepath = security.path_traversal_check(filepath, get_upload_dir()) if os.path.isfile(checked_filepath): filepath = checked_filepath - except security.SuspiciousFileOperation: + except security.SuspiciousFileOperationError: abort(400, description=_("Invalid filename")) return_data = io.BytesIO() @@ -1080,7 +1075,7 @@ def create_app(args): swag["info"]["version"] = get_version() swag["info"]["title"] = "LibreTranslate" - @app.route(API_URL) + @app.route(api_url) @limiter.exempt def spec(): return jsonify(lazy_swag(swag)) @@ -1093,14 +1088,14 @@ def create_app(args): return override_lang return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes())) - babel = Babel(app, locale_selector=get_locale) + Babel(app, locale_selector=get_locale) app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html) # Call factory function to create our blueprint - swaggerui_blueprint = get_swaggerui_blueprint(SWAGGER_URL, API_URL) + swaggerui_blueprint = get_swaggerui_blueprint(swagger_url, api_url) if args.url_prefix: - app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL) + app.register_blueprint(swaggerui_blueprint, url_prefix=swagger_url) else: app.register_blueprint(swaggerui_blueprint) diff --git a/libretranslate/detect.py b/libretranslate/detect.py index 1b43087..b9f6f2e 100644 --- a/libretranslate/detect.py +++ b/libretranslate/detect.py @@ -2,10 +2,11 @@ import pycld2 as cld2 -class UnknownLanguage(Exception): + +class UnknownLanguageError(Exception): pass -class Language(object): +class Language: def __init__(self, choice): name, code, confidence, bytesize = choice self.code = code @@ -23,7 +24,7 @@ class Language(object): return Language(("", code, 100, 0)) -class Detector(object): +class Detector: """ Detect the language used in a snippet of text.""" def __init__(self, text, quiet=False): @@ -56,17 +57,16 @@ class Detector(object): if not reliable: self.reliable = False reliable, index, top_3_choices = cld2.detect(text, bestEffort=True) - - if not self.quiet: - if not reliable: - raise UnknownLanguage("Try passing a longer snippet of text") + + if not self.quiet and not reliable: + raise UnknownLanguageError("Try passing a longer snippet of text") self.languages = [Language(x) for x in top_3_choices] self.language = self.languages[0] return self.language def __str__(self): - text = "Prediction is reliable: {}\n".format(self.reliable) - text += u"\n".join(["Language {}: {}".format(i+1, str(l)) + text = f"Prediction is reliable: {self.reliable}\n" + text += "\n".join([f"Language {i+1}: {str(l)}" for i,l in enumerate(self.languages)]) - return text \ No newline at end of file + return text diff --git a/libretranslate/init.py b/libretranslate/init.py index b6238fa..eec788d 100644 --- a/libretranslate/init.py +++ b/libretranslate/init.py @@ -1,4 +1,3 @@ -from pathlib import Path from argostranslate import package, translate @@ -46,14 +45,12 @@ def check_and_install_models(force=False, load_only_lang_codes=None): # Download and install all available packages for available_package in available_packages: print( - "Downloading %s (%s) ..." - % (available_package, available_package.package_version) + f"Downloading {available_package} ({available_package.package_version}) ..." ) available_package.install() # reload installed languages libretranslate.language.languages = translate.get_installed_languages() print( - "Loaded support for %s languages (%s models total)!" - % (len(translate.get_installed_languages()), len(available_packages)) + f"Loaded support for {len(translate.get_installed_languages())} languages ({len(available_packages)} models total)!" ) diff --git a/libretranslate/language.py b/libretranslate/language.py index 1386bad..75fd0a7 100644 --- a/libretranslate/language.py +++ b/libretranslate/language.py @@ -1,7 +1,7 @@ -import string from argostranslate import translate -from libretranslate.detect import Detector, UnknownLanguage + +from libretranslate.detect import Detector, UnknownLanguageError __languages = None @@ -29,7 +29,7 @@ def detect_languages(text): for i in range(len(d)): d[i].text_length = len(t) candidates.extend(d) - except UnknownLanguage: + except UnknownLanguageError: pass # total read bytes of the provided text @@ -83,10 +83,10 @@ def improve_translation_formatting(source, translation, improve_punctuation=True if not len(source): return "" - + if not len(translation): return source - + if improve_punctuation: source_last_char = source[len(source) - 1] translation_last_char = translation[len(translation) - 1] diff --git a/libretranslate/locales.py b/libretranslate/locales.py index 896e6fa..9c08d4c 100644 --- a/libretranslate/locales.py +++ b/libretranslate/locales.py @@ -1,10 +1,11 @@ -import os import json +import os from functools import lru_cache + from flask_babel import gettext as _ from flask_babel import lazy_gettext as _lazy +from markupsafe import Markup, escape -from markupsafe import escape, Markup @lru_cache(maxsize=None) def get_available_locales(only_reviewed=True, sort_by_name=False): diff --git a/libretranslate/main.py b/libretranslate/main.py index 0e02145..57dd723 100644 --- a/libretranslate/main.py +++ b/libretranslate/main.py @@ -197,7 +197,7 @@ def main(): from waitress import serve url_scheme = "https" if args.ssl else "http" - print("Running on %s://%s:%s%s" % (url_scheme, args.host, args.port, args.url_prefix)) + print(f"Running on {url_scheme}://{args.host}:{args.port}{args.url_prefix}") serve( app, diff --git a/libretranslate/manage.py b/libretranslate/manage.py index 68beb61..d296388 100644 --- a/libretranslate/manage.py +++ b/libretranslate/manage.py @@ -49,7 +49,7 @@ def manage(): print("There are no API keys") else: for item in keys: - print("%s: %s" % item) + print("{}: {}".format(*item)) elif args.sub_command == "add": print(db.add(args.req_limit, args.key)[0]) diff --git a/libretranslate/scheduler.py b/libretranslate/scheduler.py index 3300095..de54884 100644 --- a/libretranslate/scheduler.py +++ b/libretranslate/scheduler.py @@ -1,5 +1,7 @@ import atexit + from apscheduler.schedulers.background import BackgroundScheduler + scheduler = None def setup(args): diff --git a/libretranslate/secret.py b/libretranslate/secret.py index a50fcc7..76cf606 100644 --- a/libretranslate/secret.py +++ b/libretranslate/secret.py @@ -1,9 +1,9 @@ -import atexit import random import string from libretranslate.storage import get_storage + def generate_secret(): return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7)) diff --git a/libretranslate/security.py b/libretranslate/security.py index 99415da..7163a83 100644 --- a/libretranslate/security.py +++ b/libretranslate/security.py @@ -1,7 +1,7 @@ import os -class SuspiciousFileOperation(Exception): +class SuspiciousFileOperationError(Exception): pass @@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path): unsafe_path = os.path.abspath(unsafe_path) if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path): - raise SuspiciousFileOperation("{} is not safe".format(unsafe_path)) + raise SuspiciousFileOperationError(f"{unsafe_path} is not safe") # Passes the check - return unsafe_path \ No newline at end of file + return unsafe_path diff --git a/libretranslate/suggestions.py b/libretranslate/suggestions.py index 911b811..73944dd 100644 --- a/libretranslate/suggestions.py +++ b/libretranslate/suggestions.py @@ -1,5 +1,5 @@ -import sqlite3 import os +import sqlite3 from expiringdict import ExpiringDict @@ -10,7 +10,7 @@ class Database: def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): # Legacy check - this can be removed at some point in the near future if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"): - print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db")) + print("Migrating {} to {}".format("suggestions.db", "db/suggestions.db")) try: os.rename("suggestions.db", "db/suggestions.db") except Exception as e: diff --git a/libretranslate/tests/test_api/conftest.py b/libretranslate/tests/test_api/conftest.py index 3eb4f79..a1db034 100644 --- a/libretranslate/tests/test_api/conftest.py +++ b/libretranslate/tests/test_api/conftest.py @@ -1,4 +1,5 @@ import sys + import pytest from libretranslate.app import create_app diff --git a/libretranslate/tests/test_api/test_api_translate.py b/libretranslate/tests/test_api/test_api_translate.py index 96f8632..ae08052 100644 --- a/libretranslate/tests/test_api/test_api_translate.py +++ b/libretranslate/tests/test_api/test_api_translate.py @@ -43,7 +43,7 @@ def test_api_translate_unsupported_language(client): response_json = json.loads(response.data) assert "error" in response_json - assert "zz is not supported" == response_json["error"] + assert response_json["error"] == "zz is not supported" assert response.status_code == 400 @@ -57,5 +57,5 @@ def test_api_translate_missing_parameter(client): response_json = json.loads(response.data) assert "error" in response_json - assert "Invalid request: missing q parameter" == response_json["error"] + assert response_json["error"] == "Invalid request: missing q parameter" assert response.status_code == 400 diff --git a/libretranslate/tests/test_init.py b/libretranslate/tests/test_init.py index 198f223..cd796df 100644 --- a/libretranslate/tests/test_init.py +++ b/libretranslate/tests/test_init.py @@ -1,6 +1,7 @@ -from libretranslate.init import boot from argostranslate import package +from libretranslate.init import boot + def test_boot_argos(): """Test Argos translate models initialization""" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d7d1252 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,167 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +requires-python = ">=3.8" +name = "libretranslate" +description = "Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services." +readme = "README.md" +license = { file = "LICENSE" } +authors = [ + { name = "Piero Toffanin", email = "pt@uav4geo.com" }, + { name = "LibreTranslate Authors" }, +] +maintainers = [ + { name = "Piero Toffanin", email = "pt@uav4geo.com" }, + { name = "LibreTranslate Authors" }, +] +keywords = [ + "Python", + "Translate", + "Translation", + "API", +] +classifiers = [ + "Operating System :: OS Independent", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10" +] +dynamic = ["version"] + +dependencies = [ + "argostranslate ==1.8.0", + "Flask ==2.2.2", + "flask-swagger ==0.2.14", + "flask-swagger-ui ==4.11.1", + "Flask-Limiter ==2.6.3", + "Flask-Babel ==3.1.0", + "Flask-Session ==0.4.0", + "waitress ==2.1.2", + "expiringdict ==1.2.2", + " LTpycld2==0.42", + "morfessor ==2.0.6", + "appdirs ==1.4.4", + "APScheduler ==3.9.1", + "translatehtml ==1.5.2", + "argos-translate-files ==1.1.1", + "itsdangerous ==2.1.2", + "Werkzeug ==2.2.2", + "requests ==2.28.1", + "redis ==4.3.4", + "prometheus-client ==0.15.0", + "polib ==1.1.1", +] + +[project.scripts] +libretranslate = "libretranslate.main:main" +ltmanage = "libretranslate.manage:manage" + + +[project.optional-dependencies] +test = [ + "pytest >=7.2.0", + "pytest-cov", + "ruff ==0.0.277", + "types-requests", +] + + +[project.urls] +Homepage = "https://libretranslate.com" +Source = "https://github.com/LibreTranslate/LibreTranslate" +Documentation = "https://github.com/LibreTranslate/LibreTranslate" +Tracker = "https://github.com/LibreTranslate/LibreTranslate/issues" +History = "https://github.com/LibreTranslate/LibreTranslate/releases" + + +# ENVIRONMENTS AND SCRIPTS +[tool.hatch.envs.default] +features = [ + "test", +] + + +[tool.hatch.envs.default.scripts] +dev = "python main.py {args}" +locales = "python scripts/compile_locales.py" +fmt = [ + "ruff libretranslate scripts --fix", +] +test = [ + "fmt", + "pytest {args}", +] +cov = [ + "pytest --cov-report html {args}", + "python -c 'import webbrowser; webbrowser.open(\"http://0.0.0.0:3000\")'", + "python -m http.server 3000 --directory ./htmlcov", +] + + +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11"] + + +# TOOLS +[tool.hatch.version] +path = "VERSION" +pattern = "^(?P[0-9]*.[0-9]*.[0-9]*)$" + + +[tool.pytest.ini_options] +addopts = [ + "-v", + "--cov=libretranslate", + "--color=yes", + "--cov-report=term-missing", +] + + +# https://beta.ruff.rs/docs/rules +[tool.ruff] +src = ["libretranslate", "scripts"] +target-version = "py38" +line-length = 136 +select = [ + "I", # isort + "N", # pep8-naming + "S", # bandit + "A", # flake8-builtins + "YTT", # flake8-2020 + "B", # flake8-bugbear + # "C", # flake8-comprehensions + "ICN", # flake8-import-conventions + # "SIM", # flake8-simplify + "TID", # flake8-tidy-imports + # "Q", # flake8-quotes + # "FBT", # flake8-boolean-trap + "F", # pyflakes + "UP", # pyupgrade + # "E", # pycodestyle errors + # "W", # pycodestyle warnings + # "PLC", # pylint convention + "PLE", # pylint error + # "PLR", # pylint refactor + # "PLW", # pylint warning + # "RUF", # ruff specific +] + +ignore = [ + "E501", # line too long + "A003", # Class attribute is shadowing a python builtin + "S101", # Use of `assert` detected + "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes + "T201", "T203", # remove print and pprint + "E402", # Module level import not at top of file +] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["I", "F401"] # module imported but unused + + +[tool.ruff.mccabe] +max-complexity = 12 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2948d5e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -argostranslate==1.8.0 -Flask==2.2.2 -flask-swagger==0.2.14 -flask-swagger-ui==4.11.1 -Flask-Limiter==2.6.3 -Flask-Babel==3.1.0 -Flask-Session==0.4.0 -waitress==2.1.2 -expiringdict==1.2.2 -LTpycld2==0.42 -morfessor==2.0.6 -appdirs==1.4.4 -APScheduler==3.9.1 -translatehtml==1.5.2 -argos-translate-files==1.1.1 -itsdangerous==2.1.2 -Werkzeug==2.2.2 -requests==2.28.1 -redis==4.3.4 -prometheus-client==0.15.0 -polib==1.1.1 diff --git a/scripts/compile_locales.py b/scripts/compile_locales.py index 5aff9af..2920354 100755 --- a/scripts/compile_locales.py +++ b/scripts/compile_locales.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from babel.messages.frontend import main as pybabel @@ -15,7 +16,7 @@ if __name__ == "__main__": link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code'] if l['code'] == 'en': link = "https://hosted.weblate.org/projects/libretranslate/app/" - print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link)) + print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link)) else: locales_dir = os.path.join("libretranslate", "locales") if not os.path.isdir(locales_dir): diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index 845da5b..8c4f709 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -1,7 +1,9 @@ -from prometheus_client import multiprocess import re import sys +from prometheus_client import multiprocess + + def child_exit(server, worker): multiprocess.mark_process_dead(worker.pid) @@ -35,7 +37,7 @@ def on_starting(server): args = get_args() - from libretranslate import storage, scheduler, flood, secret + from libretranslate import flood, scheduler, secret, storage storage.setup(args.shared_storage) scheduler.setup(args) flood.setup(args) diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py index cd62b4f..07bcaa4 100644 --- a/scripts/healthcheck.py +++ b/scripts/healthcheck.py @@ -1,4 +1,5 @@ import requests + response = requests.post( url='http://0.0.0.0:5000/translate', headers={'Content-Type': 'application/json'}, @@ -6,6 +7,7 @@ response = requests.post( 'q': 'Hello World!', 'source': 'en', 'target': 'en' - } + }, + timeout=60 ) # if server unavailable then requests with raise exception and healthcheck will fail diff --git a/scripts/install_models.py b/scripts/install_models.py index d844afb..2d3d313 100755 --- a/scripts/install_models.py +++ b/scripts/install_models.py @@ -1,8 +1,10 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import argparse + from libretranslate.init import check_and_install_models if __name__ == "__main__": diff --git a/scripts/suggestions-to-jsonl.py b/scripts/suggestions-to-jsonl.py index 6cd8e62..7492ac0 100755 --- a/scripts/suggestions-to-jsonl.py +++ b/scripts/suggestions-to-jsonl.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import argparse -import time -import sqlite3 import json +import sqlite3 +import time if __name__ == "__main__": parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db") diff --git a/scripts/update_locales.py b/scripts/update_locales.py index 1fd2853..93c35fa 100755 --- a/scripts/update_locales.py +++ b/scripts/update_locales.py @@ -1,17 +1,19 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import re -import polib import json +import re + +import polib from babel.messages.frontend import main as pybabel -from libretranslate.language import load_languages, improve_translation_formatting -from libretranslate.locales import get_available_locale_codes, swag_eval -from translatehtml import translate_html -from libretranslate.app import get_version, create_app -from libretranslate.main import get_args from flask_swagger import swagger +from libretranslate.app import create_app, get_version +from libretranslate.language import improve_translation_formatting, load_languages +from libretranslate.locales import get_available_locale_codes, swag_eval +from libretranslate.main import get_args +from translatehtml import translate_html # Update strings if __name__ == "__main__": @@ -74,7 +76,7 @@ if __name__ == "__main__": if not os.path.isfile(meta_file): with open(meta_file, 'w') as f: f.write(json.dumps({ - 'name': next((lang.name for lang in languages if lang.code == l)), + 'name': next(lang.name for lang in languages if lang.code == l), 'reviewed': False }, indent=4)) print("Wrote %s" % meta_file) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e7ca7e4..0000000 --- a/setup.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[flake8] -exclude = .git, - .vscode, - .gitignore, - README.md, - venv, - test, - setup.py, - libretranslate/__init__.py - -max-line-length = 136 -ignore = E741 diff --git a/setup.py b/setup.py deleted file mode 100644 index aaf2f88..0000000 --- a/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from setuptools import setup, find_packages - -setup( - version=open('VERSION').read().strip(), - name='libretranslate', - license='GNU Affero General Public License v3.0', - description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.', - author='LibreTranslate Authors', - author_email='pt@uav4geo.com', - url='https://libretranslate.com', - packages=find_packages(), - # packages=find_packages(include=['openpredict']), - # package_dir={'openpredict': 'openpredict'}, - package_data={'': ['static/*', 'static/**/*', 'templates/*', 'locales/**/meta.json', 'locales/**/**/*.mo']}, - include_package_data=True, - entry_points={ - 'console_scripts': [ - 'libretranslate=libretranslate.main:main', - 'ltmanage=libretranslate.manage:manage' - ], - }, - - python_requires='>=3.8.0', - long_description=open('README.md').read(), - long_description_content_type="text/markdown", - install_requires=open("requirements.txt", "r").readlines(), - tests_require=['pytest==7.2.0'], - setup_requires=['pytest-runner'], - classifiers=[ - "License :: OSI Approved :: GNU Affero General Public License v3 ", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10" - ] -)