From f0943dada1530ebae8ca516017e6a5b674a495bf Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 9 Jul 2023 11:35:53 +0200 Subject: [PATCH 1/8] Upgrade project config to use a pyproject.toml file for the package metadata and tool configs (PEP 621), and the hatch build-system. Remove requirements.txt and setup.py --- pyproject.toml | 191 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 21 ------ setup.py | 38 ---------- 3 files changed, 191 insertions(+), 59 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..10b56c2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,191 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +requires-python = ">=3.8" +name = "libretranslate" +description = "Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services." +readme = "README.md" +license = { file = "LICENSE" } +authors = [ + { name = "Piero Toffanin", email = "pt@uav4geo.com" }, + { name = "LibreTranslate Authors" }, +] +maintainers = [ + { name = "Piero Toffanin", email = "pt@uav4geo.com" }, + { name = "LibreTranslate Authors" }, +] +keywords = [ + "Python", + "Translate", + "Translation", +] +classifiers = [ + "Operating System :: OS Independent", + "License :: OSI Approved :: GNU Affero General Public License v3 ", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10" +] +dynamic = ["version"] + +dependencies = [ + "argostranslate ==1.8.0", + # "argos-translate-files", + "Flask ==2.2.2", + "flask-swagger ==0.2.14", + "flask-swagger-ui ==4.11.1", + "Flask-Limiter ==2.6.3", + "Flask-Babel ==3.1.0", + "Flask-Session ==0.4.0", + "waitress ==2.1.2", + "expiringdict ==1.2.2", + " LTpycld2==0.42", + "morfessor ==2.0.6", + "appdirs ==1.4.4", + "APScheduler ==3.9.1", + "translatehtml ==1.5.2", + "argos-translate-files ==1.1.1", + "itsdangerous ==2.1.2", + "Werkzeug ==2.2.2", + "requests ==2.28.1", + "redis ==4.3.4", + "prometheus-client ==0.15.0", + "polib ==1.1.1", +] +# package_data={'': ['static/*', 'static/**/*', 'templates/*', 'locales/**/meta.json', 'locales/**/**/*.mo']}, + +[project.scripts] +libretranslate = "libretranslate.main:main" +ltmanage = "libretranslate.manage:manage" + + +[project.optional-dependencies] +test = [ + "pytest >=7.2.0", + "pytest-runner", + "pytest-cov", + # "mypy >=1.4.1", + "types-requests", +] + + +[project.urls] +Homepage = "https://libretranslate.com" +Source = "https://github.com/LibreTranslate/LibreTranslate" +Documentation = "https://github.com/LibreTranslate/LibreTranslate" +Tracker = "https://github.com/LibreTranslate/LibreTranslate/issues" +History = "https://github.com/LibreTranslate/LibreTranslate/releases" + + +# ENVIRONMENTS AND SCRIPTS +[tool.hatch.envs.default] +features = [ + "test", +] + +[tool.hatch.envs.default.scripts] +dev = "python main.py {args}" +lint = [ + "flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics", + "flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", + # "ruff --fix", +] +fmt = [ + "ruff --fix", + # "mypy", +] +test = [ + "pytest {args}", +] +cov = [ + "pytest --cov-report html {args}", + "python -c 'import webbrowser; webbrowser.open(\"http://0.0.0.0:3000\")'", + "python -m http.server 3000 --directory ./htmlcov", +] + + +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11"] + + +# TOOLS +[tool.hatch.version] +path = "VERSION" +pattern = "^([0-9]*.[0-9]*.[0-9]*)$" + + +[tool.pytest.ini_options] +addopts = [ + "-v", + "--cov=libretranslate", + "--color=yes", + "--cov-report=term-missing", +] + + +# https://github.com/charliermarsh/ruff#supported-rules +[tool.ruff] +src = ["libretranslate", "scripts"] +target-version = "py38" +line-length = 136 +select = [ + "A", # flake8-builtins + "B", # flake8-bugbear + "C", # flake8-comprehensions + "E", # pycodestyle errors + "F", # pyflakes + # "FBT", # flake8-boolean-trap + "I", # isort + "ICN", # flake8-import-conventions + "N", # pep8-naming + "PLC", # pylint convention + "PLE", # pylint error + # "PLR", # pylint refactor Magic value used in comparison, consider replacing 400 with a constant variable + "PLW", # pylint warning + "Q", # flake8-quotes + "RUF", # ruff specific + "S", # bandit + "SIM", # flake8-simplify + "T", + "TID", # flake8-tidy-imports + "UP", # pyupgrade + "W", # pycodestyle warnings + "YTT", # flake8-2020 +] + +ignore = [ + # "E741", + # "B008", # do not perform function calls in argument defaults (required for FastAPI afaik) + # "E501", # line too long + # "C901", # too complex + # "S101", # Use of `assert` detected + # "T201", "T203", # remove print and pprint +] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["I", "F401"] # module imported but unused + + +[tool.ruff.mccabe] +max-complexity = 10 + +# [flake8] ignore = E741 + + +# [tool.mypy] +# files = ["src/"] +# strict = true +# implicit_reexport = true +# follow_imports = "normal" +# ignore_missing_imports = true +# pretty = true +# show_column_numbers = true +# warn_no_return = true +# warn_unused_ignores = true +# warn_redundant_casts = true +# disallow_untyped_defs = true +# disallow_any_generics = true +# disallow_untyped_calls = false # needed due to _eval() not being typed in rdflib diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2948d5e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -argostranslate==1.8.0 -Flask==2.2.2 -flask-swagger==0.2.14 -flask-swagger-ui==4.11.1 -Flask-Limiter==2.6.3 -Flask-Babel==3.1.0 -Flask-Session==0.4.0 -waitress==2.1.2 -expiringdict==1.2.2 -LTpycld2==0.42 -morfessor==2.0.6 -appdirs==1.4.4 -APScheduler==3.9.1 -translatehtml==1.5.2 -argos-translate-files==1.1.1 -itsdangerous==2.1.2 -Werkzeug==2.2.2 -requests==2.28.1 -redis==4.3.4 -prometheus-client==0.15.0 -polib==1.1.1 diff --git a/setup.py b/setup.py deleted file mode 100644 index aaf2f88..0000000 --- a/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from setuptools import setup, find_packages - -setup( - version=open('VERSION').read().strip(), - name='libretranslate', - license='GNU Affero General Public License v3.0', - description='Free and Open Source Machine Translation API. Self-hosted, no limits, no ties to proprietary services.', - author='LibreTranslate Authors', - author_email='pt@uav4geo.com', - url='https://libretranslate.com', - packages=find_packages(), - # packages=find_packages(include=['openpredict']), - # package_dir={'openpredict': 'openpredict'}, - package_data={'': ['static/*', 'static/**/*', 'templates/*', 'locales/**/meta.json', 'locales/**/**/*.mo']}, - include_package_data=True, - entry_points={ - 'console_scripts': [ - 'libretranslate=libretranslate.main:main', - 'ltmanage=libretranslate.manage:manage' - ], - }, - - python_requires='>=3.8.0', - long_description=open('README.md').read(), - long_description_content_type="text/markdown", - install_requires=open("requirements.txt", "r").readlines(), - tests_require=['pytest==7.2.0'], - setup_requires=['pytest-runner'], - classifiers=[ - "License :: OSI Approved :: GNU Affero General Public License v3 ", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10" - ] -) From 19fb149112618c024e0c803946759bfb667d316e Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 9 Jul 2023 11:43:58 +0200 Subject: [PATCH 2/8] Update checkout and setup-python actions version in workflow and improve ruff config --- .github/workflows/publish-docker.yml | 2 +- .github/workflows/publish-package.yml | 8 ++++---- .github/workflows/run-tests.yml | 10 ++++------ pyproject.toml | 15 ++++++++------- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index d2404cf..a2b5ebb 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index b183d08..5b752ba 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -15,9 +15,9 @@ jobs: python-version: ['3.8', '3.9', '3.10'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -45,9 +45,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - name: Install dependencies diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 9e59bfd..a14d001 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,17 +21,15 @@ jobs: python-version: ['3.8', '3.9', '3.10'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pytest flake8 - pip install . + pip install ".[test]" - name: Check code style with flake8 (lint) run: | @@ -49,7 +47,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Docker build run: docker build -f docker/Dockerfile -t libretranslate . diff --git a/pyproject.toml b/pyproject.toml index 10b56c2..fb90855 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,6 @@ dependencies = [ "prometheus-client ==0.15.0", "polib ==1.1.1", ] -# package_data={'': ['static/*', 'static/**/*', 'templates/*', 'locales/**/meta.json', 'locales/**/**/*.mo']}, [project.scripts] libretranslate = "libretranslate.main:main" @@ -65,8 +64,9 @@ ltmanage = "libretranslate.manage:manage" [project.optional-dependencies] test = [ "pytest >=7.2.0", - "pytest-runner", "pytest-cov", + "flake8", + # "pytest-runner", # "mypy >=1.4.1", "types-requests", ] @@ -94,7 +94,7 @@ lint = [ # "ruff --fix", ] fmt = [ - "ruff --fix", + "ruff libretranslate scripts --fix", # "mypy", ] test = [ @@ -159,10 +159,11 @@ select = [ ignore = [ # "E741", # "B008", # do not perform function calls in argument defaults (required for FastAPI afaik) - # "E501", # line too long + "E501", # line too long # "C901", # too complex - # "S101", # Use of `assert` detected - # "T201", "T203", # remove print and pprint + "S101", # Use of `assert` detected + "T201", "T203", # remove print and pprint + "E402", # Module level import not at top of file ] [tool.ruff.per-file-ignores] @@ -170,7 +171,7 @@ ignore = [ [tool.ruff.mccabe] -max-complexity = 10 +max-complexity = 12 # [flake8] ignore = E741 From 0e404b5ee3cb5556be4ba9c2eada5c4e199c3830 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 9 Jul 2023 12:18:58 +0200 Subject: [PATCH 3/8] update contributing to provide instructions to run script using hatch --- CONTRIBUTING.md | 46 ++++++++++++++++++++++++++++++++++++--- pyproject.toml | 57 ++++++++++++++++--------------------------------- 2 files changed, 61 insertions(+), 42 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9621360..9b04ea3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,17 +20,57 @@ sudo dnf install cmake ## Getting Started +Install `hatch` to manage the projects dependencies and run dev scripts: + +```bash +pipx install hatch +``` + +Clone the repository: + ```bash git clone https://github.com/LibreTranslate/LibreTranslate.git cd LibreTranslate -pip install -e . -libretranslate [args] +``` +Run in development: + +```bash +hatch run dev +``` + +Then open a web browser to + +You can also start a new shell in a virtual environment with libretranslate installed: + +```bash +hatch shell +libretranslate [args] # Or python main.py [args] ``` -Then open a web browser to +> You can still use `pip install -e ".[test]"` directly if you don't want to use hatch. + +## Run the tests + +Run the test suite and linting checks: + +```bash +hatch run test +``` + +To display all `print()` when debugging: + +```bash +hatch run test -s +``` + +You can also run the tests on multiple python versions: + +```bash +hatch run all:test +``` ## Run with Docker diff --git a/pyproject.toml b/pyproject.toml index fb90855..66d6598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ keywords = [ ] classifiers = [ "Operating System :: OS Independent", - "License :: OSI Approved :: GNU Affero General Public License v3 ", + "License :: OSI Approved :: GNU Affero General Public License v3", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -33,7 +33,6 @@ dynamic = ["version"] dependencies = [ "argostranslate ==1.8.0", - # "argos-translate-files", "Flask ==2.2.2", "flask-swagger ==0.2.14", "flask-swagger-ui ==4.11.1", @@ -66,9 +65,8 @@ test = [ "pytest >=7.2.0", "pytest-cov", "flake8", - # "pytest-runner", - # "mypy >=1.4.1", "types-requests", + # "mypy >=1.4.1", ] @@ -91,11 +89,10 @@ dev = "python main.py {args}" lint = [ "flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics", "flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", - # "ruff --fix", + # "ruff libretranslate scripts --fix", ] fmt = [ "ruff libretranslate scripts --fix", - # "mypy", ] test = [ "pytest {args}", @@ -114,7 +111,7 @@ python = ["3.8", "3.9", "3.10", "3.11"] # TOOLS [tool.hatch.version] path = "VERSION" -pattern = "^([0-9]*.[0-9]*.[0-9]*)$" +pattern = "^(?P[0-9]*.[0-9]*.[0-9]*)$" [tool.pytest.ini_options] @@ -132,32 +129,32 @@ src = ["libretranslate", "scripts"] target-version = "py38" line-length = 136 select = [ + "I", # isort + "N", # pep8-naming + "S", # bandit "A", # flake8-builtins + "YTT", # flake8-2020 "B", # flake8-bugbear "C", # flake8-comprehensions - "E", # pycodestyle errors - "F", # pyflakes - # "FBT", # flake8-boolean-trap - "I", # isort "ICN", # flake8-import-conventions - "N", # pep8-naming + "SIM", # flake8-simplify + "TID", # flake8-tidy-imports + "Q", # flake8-quotes + "FBT", # flake8-boolean-trap + "F", # pyflakes + "UP", # pyupgrade + "E", # pycodestyle errors + "W", # pycodestyle warnings "PLC", # pylint convention "PLE", # pylint error - # "PLR", # pylint refactor Magic value used in comparison, consider replacing 400 with a constant variable + "PLR", # pylint refactor "PLW", # pylint warning - "Q", # flake8-quotes "RUF", # ruff specific - "S", # bandit - "SIM", # flake8-simplify "T", - "TID", # flake8-tidy-imports - "UP", # pyupgrade - "W", # pycodestyle warnings - "YTT", # flake8-2020 ] ignore = [ - # "E741", + # "E741", # From original flake8 ignore # "B008", # do not perform function calls in argument defaults (required for FastAPI afaik) "E501", # line too long # "C901", # too complex @@ -172,21 +169,3 @@ ignore = [ [tool.ruff.mccabe] max-complexity = 12 - -# [flake8] ignore = E741 - - -# [tool.mypy] -# files = ["src/"] -# strict = true -# implicit_reexport = true -# follow_imports = "normal" -# ignore_missing_imports = true -# pretty = true -# show_column_numbers = true -# warn_no_return = true -# warn_unused_ignores = true -# warn_redundant_casts = true -# disallow_untyped_defs = true -# disallow_any_generics = true -# disallow_untyped_calls = false # needed due to _eval() not being typed in rdflib From 62712dcad37d64d19a705fc94941c541ae8fa564 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 9 Jul 2023 12:23:48 +0200 Subject: [PATCH 4/8] improve ruff config --- pyproject.toml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66d6598..7eeab4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,14 +87,15 @@ features = [ [tool.hatch.envs.default.scripts] dev = "python main.py {args}" lint = [ - "flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics", - "flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", - # "ruff libretranslate scripts --fix", + # "flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics", + # "flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", + "ruff libretranslate scripts", ] fmt = [ "ruff libretranslate scripts --fix", ] test = [ + # "fmt", "pytest {args}", ] cov = [ @@ -135,19 +136,19 @@ select = [ "A", # flake8-builtins "YTT", # flake8-2020 "B", # flake8-bugbear - "C", # flake8-comprehensions + # "C", # flake8-comprehensions "ICN", # flake8-import-conventions "SIM", # flake8-simplify "TID", # flake8-tidy-imports - "Q", # flake8-quotes + # "Q", # flake8-quotes "FBT", # flake8-boolean-trap "F", # pyflakes "UP", # pyupgrade - "E", # pycodestyle errors - "W", # pycodestyle warnings + # "E", # pycodestyle errors + # "W", # pycodestyle warnings "PLC", # pylint convention "PLE", # pylint error - "PLR", # pylint refactor + # "PLR", # pylint refactor "PLW", # pylint warning "RUF", # ruff specific "T", From bf18dcbcf923bfa4894969ee02c5600abef861e5 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 9 Jul 2023 12:29:11 +0200 Subject: [PATCH 5/8] run ruff formatting --- libretranslate/api_keys.py | 11 ++-- libretranslate/app.py | 51 +++++++++---------- libretranslate/detect.py | 14 ++--- libretranslate/init.py | 1 - libretranslate/language.py | 2 +- libretranslate/locales.py | 5 +- libretranslate/main.py | 2 +- libretranslate/manage.py | 2 +- libretranslate/scheduler.py | 2 + libretranslate/secret.py | 2 +- libretranslate/security.py | 2 +- libretranslate/suggestions.py | 4 +- libretranslate/tests/test_api/conftest.py | 1 + .../tests/test_api/test_api_translate.py | 4 +- libretranslate/tests/test_init.py | 3 +- pyproject.toml | 11 ++-- scripts/compile_locales.py | 5 +- scripts/gunicorn_conf.py | 6 ++- scripts/healthcheck.py | 1 + scripts/install_models.py | 4 +- scripts/suggestions-to-jsonl.py | 4 +- scripts/update_locales.py | 20 ++++---- 22 files changed, 82 insertions(+), 75 deletions(-) diff --git a/libretranslate/api_keys.py b/libretranslate/api_keys.py index dc1da3e..0583531 100644 --- a/libretranslate/api_keys.py +++ b/libretranslate/api_keys.py @@ -1,8 +1,10 @@ import os import sqlite3 import uuid + import requests from expiringdict import ExpiringDict + from libretranslate.default_values import DEFAULT_ARGUMENTS as DEFARGS DEFAULT_DB_PATH = DEFARGS['API_KEYS_DB_PATH'] @@ -12,14 +14,14 @@ class Database: def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): # Legacy check - this can be removed at some point in the near future if os.path.isfile("api_keys.db") and not os.path.isfile("db/api_keys.db"): - print("Migrating %s to %s" % ("api_keys.db", "db/api_keys.db")) + print("Migrating {} to {}".format("api_keys.db", "db/api_keys.db")) try: os.rename("api_keys.db", "db/api_keys.db") except Exception as e: print(str(e)) db_dir = os.path.dirname(db_path) - if not db_dir == "" and not os.path.exists(db_dir): + if db_dir != '' and not os.path.exists(db_dir): os.makedirs(db_dir) self.db_path = db_path self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age) @@ -91,10 +93,7 @@ class RemoteDatabase: print("Cannot authenticate API key: " + str(e)) return None - if res.get('error', None) is None: - req_limit = res.get('req_limit', None) - else: - req_limit = None + req_limit = res.get('req_limit', None) if res.get('error', None) is None else None self.cache[api_key] = req_limit return req_limit diff --git a/libretranslate/app.py b/libretranslate/app.py index c5fe485..77197f2 100644 --- a/libretranslate/app.py +++ b/libretranslate/app.py @@ -1,30 +1,37 @@ import io import os -import tempfile import re +import tempfile import uuid +from datetime import datetime from functools import wraps from html import unescape from timeit import default_timer -from datetime import datetime import argostranslatefiles from argostranslatefiles import get_supported_formats -from flask import (abort, Blueprint, Flask, jsonify, render_template, request, - Response, send_file, url_for, session) +from flask import Blueprint, Flask, Response, abort, jsonify, render_template, request, send_file, session, url_for +from flask_babel import Babel +from flask_session import Session from flask_swagger import swagger from flask_swagger_ui import get_swaggerui_blueprint -from flask_session import Session from translatehtml import translate_html -from werkzeug.utils import secure_filename from werkzeug.exceptions import HTTPException from werkzeug.http import http_date -from flask_babel import Babel +from werkzeug.utils import secure_filename -from libretranslate import scheduler, flood, secret, remove_translated_files, security, storage +from libretranslate import flood, remove_translated_files, scheduler, secret, security, storage from libretranslate.language import detect_languages, improve_translation_formatting -from libretranslate.locales import (_, _lazy, get_available_locales, get_available_locale_codes, gettext_escaped, - gettext_html, lazy_swag, get_alternate_locale_links) +from libretranslate.locales import ( + _, + _lazy, + get_alternate_locale_links, + get_available_locale_codes, + get_available_locales, + gettext_escaped, + gettext_html, + lazy_swag, +) from .api_keys import Database, RemoteDatabase from .suggestions import Database as SuggestionsDatabase @@ -150,10 +157,7 @@ def create_app(args): frontend_argos_language_source = languages[0] - if len(languages) >= 2: - language_target_fallback = languages[1] - else: - language_target_fallback = languages[0] + language_target_fallback = languages[1] if len(languages) >= 2 else languages[0] if args.frontend_language_target == "locale": def resolve_language_locale(): @@ -185,10 +189,7 @@ def create_app(args): if args.req_limit > 0 or args.api_keys or args.daily_req_limit > 0: api_keys_db = None if args.api_keys: - if args.api_keys_remote: - api_keys_db = RemoteDatabase(args.api_keys_remote) - else: - api_keys_db = Database(args.api_keys_db_path) + api_keys_db = RemoteDatabase(args.api_keys_remote) if args.api_keys_remote else Database(args.api_keys_db_path) from flask_limiter import Limiter @@ -220,7 +221,7 @@ def create_app(args): os.mkdir(default_mp_dir) os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_mp_dir - from prometheus_client import CONTENT_TYPE_LATEST, Summary, Gauge, CollectorRegistry, multiprocess, generate_latest + from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Gauge, Summary, generate_latest, multiprocess @bp.route("/metrics") @limiter.exempt @@ -544,10 +545,7 @@ def create_app(args): ) if args.char_limit != -1: - if batch: - chars = sum([len(text) for text in q]) - else: - chars = len(q) + chars = sum([len(text) for text in q]) if batch else len(q) if args.char_limit < chars: abort( @@ -557,10 +555,7 @@ def create_app(args): if source_lang == "auto": source_langs = [] - if batch: - auto_detect_texts = q - else: - auto_detect_texts = [q] + auto_detect_texts = q if batch else [q] overall_candidates = detect_languages(q) @@ -1093,7 +1088,7 @@ def create_app(args): return override_lang return session.get('preferred_lang', request.accept_languages.best_match(get_available_locale_codes())) - babel = Babel(app, locale_selector=get_locale) + Babel(app, locale_selector=get_locale) app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html) diff --git a/libretranslate/detect.py b/libretranslate/detect.py index 1b43087..8717658 100644 --- a/libretranslate/detect.py +++ b/libretranslate/detect.py @@ -2,10 +2,11 @@ import pycld2 as cld2 + class UnknownLanguage(Exception): pass -class Language(object): +class Language: def __init__(self, choice): name, code, confidence, bytesize = choice self.code = code @@ -23,7 +24,7 @@ class Language(object): return Language(("", code, 100, 0)) -class Detector(object): +class Detector: """ Detect the language used in a snippet of text.""" def __init__(self, text, quiet=False): @@ -57,16 +58,15 @@ class Detector(object): self.reliable = False reliable, index, top_3_choices = cld2.detect(text, bestEffort=True) - if not self.quiet: - if not reliable: - raise UnknownLanguage("Try passing a longer snippet of text") + if not self.quiet and not reliable: + raise UnknownLanguage("Try passing a longer snippet of text") self.languages = [Language(x) for x in top_3_choices] self.language = self.languages[0] return self.language def __str__(self): - text = "Prediction is reliable: {}\n".format(self.reliable) - text += u"\n".join(["Language {}: {}".format(i+1, str(l)) + text = f"Prediction is reliable: {self.reliable}\n" + text += "\n".join([f"Language {i+1}: {str(l)}" for i,l in enumerate(self.languages)]) return text \ No newline at end of file diff --git a/libretranslate/init.py b/libretranslate/init.py index b6238fa..04aa866 100644 --- a/libretranslate/init.py +++ b/libretranslate/init.py @@ -1,4 +1,3 @@ -from pathlib import Path from argostranslate import package, translate diff --git a/libretranslate/language.py b/libretranslate/language.py index 1386bad..68e3e48 100644 --- a/libretranslate/language.py +++ b/libretranslate/language.py @@ -1,6 +1,6 @@ -import string from argostranslate import translate + from libretranslate.detect import Detector, UnknownLanguage __languages = None diff --git a/libretranslate/locales.py b/libretranslate/locales.py index 896e6fa..9c08d4c 100644 --- a/libretranslate/locales.py +++ b/libretranslate/locales.py @@ -1,10 +1,11 @@ -import os import json +import os from functools import lru_cache + from flask_babel import gettext as _ from flask_babel import lazy_gettext as _lazy +from markupsafe import Markup, escape -from markupsafe import escape, Markup @lru_cache(maxsize=None) def get_available_locales(only_reviewed=True, sort_by_name=False): diff --git a/libretranslate/main.py b/libretranslate/main.py index 0e02145..57dd723 100644 --- a/libretranslate/main.py +++ b/libretranslate/main.py @@ -197,7 +197,7 @@ def main(): from waitress import serve url_scheme = "https" if args.ssl else "http" - print("Running on %s://%s:%s%s" % (url_scheme, args.host, args.port, args.url_prefix)) + print(f"Running on {url_scheme}://{args.host}:{args.port}{args.url_prefix}") serve( app, diff --git a/libretranslate/manage.py b/libretranslate/manage.py index 68beb61..d296388 100644 --- a/libretranslate/manage.py +++ b/libretranslate/manage.py @@ -49,7 +49,7 @@ def manage(): print("There are no API keys") else: for item in keys: - print("%s: %s" % item) + print("{}: {}".format(*item)) elif args.sub_command == "add": print(db.add(args.req_limit, args.key)[0]) diff --git a/libretranslate/scheduler.py b/libretranslate/scheduler.py index 3300095..de54884 100644 --- a/libretranslate/scheduler.py +++ b/libretranslate/scheduler.py @@ -1,5 +1,7 @@ import atexit + from apscheduler.schedulers.background import BackgroundScheduler + scheduler = None def setup(args): diff --git a/libretranslate/secret.py b/libretranslate/secret.py index a50fcc7..76cf606 100644 --- a/libretranslate/secret.py +++ b/libretranslate/secret.py @@ -1,9 +1,9 @@ -import atexit import random import string from libretranslate.storage import get_storage + def generate_secret(): return ''.join(random.choices(string.ascii_uppercase + string.digits, k=7)) diff --git a/libretranslate/security.py b/libretranslate/security.py index 99415da..3c4023a 100644 --- a/libretranslate/security.py +++ b/libretranslate/security.py @@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path): unsafe_path = os.path.abspath(unsafe_path) if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path): - raise SuspiciousFileOperation("{} is not safe".format(unsafe_path)) + raise SuspiciousFileOperation(f"{unsafe_path} is not safe") # Passes the check return unsafe_path \ No newline at end of file diff --git a/libretranslate/suggestions.py b/libretranslate/suggestions.py index 911b811..73944dd 100644 --- a/libretranslate/suggestions.py +++ b/libretranslate/suggestions.py @@ -1,5 +1,5 @@ -import sqlite3 import os +import sqlite3 from expiringdict import ExpiringDict @@ -10,7 +10,7 @@ class Database: def __init__(self, db_path=DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30): # Legacy check - this can be removed at some point in the near future if os.path.isfile("suggestions.db") and not os.path.isfile("db/suggestions.db"): - print("Migrating %s to %s" % ("suggestions.db", "db/suggestions.db")) + print("Migrating {} to {}".format("suggestions.db", "db/suggestions.db")) try: os.rename("suggestions.db", "db/suggestions.db") except Exception as e: diff --git a/libretranslate/tests/test_api/conftest.py b/libretranslate/tests/test_api/conftest.py index 3eb4f79..a1db034 100644 --- a/libretranslate/tests/test_api/conftest.py +++ b/libretranslate/tests/test_api/conftest.py @@ -1,4 +1,5 @@ import sys + import pytest from libretranslate.app import create_app diff --git a/libretranslate/tests/test_api/test_api_translate.py b/libretranslate/tests/test_api/test_api_translate.py index 96f8632..ae08052 100644 --- a/libretranslate/tests/test_api/test_api_translate.py +++ b/libretranslate/tests/test_api/test_api_translate.py @@ -43,7 +43,7 @@ def test_api_translate_unsupported_language(client): response_json = json.loads(response.data) assert "error" in response_json - assert "zz is not supported" == response_json["error"] + assert response_json["error"] == "zz is not supported" assert response.status_code == 400 @@ -57,5 +57,5 @@ def test_api_translate_missing_parameter(client): response_json = json.loads(response.data) assert "error" in response_json - assert "Invalid request: missing q parameter" == response_json["error"] + assert response_json["error"] == "Invalid request: missing q parameter" assert response.status_code == 400 diff --git a/libretranslate/tests/test_init.py b/libretranslate/tests/test_init.py index 198f223..cd796df 100644 --- a/libretranslate/tests/test_init.py +++ b/libretranslate/tests/test_init.py @@ -1,6 +1,7 @@ -from libretranslate.init import boot from argostranslate import package +from libretranslate.init import boot + def test_boot_argos(): """Test Argos translate models initialization""" diff --git a/pyproject.toml b/pyproject.toml index 7eeab4b..e9eaed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,18 +138,18 @@ select = [ "B", # flake8-bugbear # "C", # flake8-comprehensions "ICN", # flake8-import-conventions - "SIM", # flake8-simplify + # "SIM", # flake8-simplify "TID", # flake8-tidy-imports # "Q", # flake8-quotes - "FBT", # flake8-boolean-trap + # "FBT", # flake8-boolean-trap "F", # pyflakes "UP", # pyupgrade # "E", # pycodestyle errors # "W", # pycodestyle warnings - "PLC", # pylint convention + # "PLC", # pylint convention "PLE", # pylint error # "PLR", # pylint refactor - "PLW", # pylint warning + # "PLW", # pylint warning "RUF", # ruff specific "T", ] @@ -158,8 +158,9 @@ ignore = [ # "E741", # From original flake8 ignore # "B008", # do not perform function calls in argument defaults (required for FastAPI afaik) "E501", # line too long - # "C901", # too complex + "A003", # Class attribute is shadowing a python builtin "S101", # Use of `assert` detected + "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes "T201", "T203", # remove print and pprint "E402", # Module level import not at top of file ] diff --git a/scripts/compile_locales.py b/scripts/compile_locales.py index 5aff9af..2920354 100755 --- a/scripts/compile_locales.py +++ b/scripts/compile_locales.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from babel.messages.frontend import main as pybabel @@ -15,7 +16,7 @@ if __name__ == "__main__": link = "https://hosted.weblate.org/translate/libretranslate/app/%s/" % l['code'] if l['code'] == 'en': link = "https://hosted.weblate.org/projects/libretranslate/app/" - print("%s | %s | %s" % (l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link)) + print("{} | {} | {}".format(l['name'], ':heavy_check_mark:' if l['reviewed'] else '', "[Edit](%s)" % link)) else: locales_dir = os.path.join("libretranslate", "locales") if not os.path.isdir(locales_dir): diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index 845da5b..8c4f709 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -1,7 +1,9 @@ -from prometheus_client import multiprocess import re import sys +from prometheus_client import multiprocess + + def child_exit(server, worker): multiprocess.mark_process_dead(worker.pid) @@ -35,7 +37,7 @@ def on_starting(server): args = get_args() - from libretranslate import storage, scheduler, flood, secret + from libretranslate import flood, scheduler, secret, storage storage.setup(args.shared_storage) scheduler.setup(args) flood.setup(args) diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py index cd62b4f..20eba19 100644 --- a/scripts/healthcheck.py +++ b/scripts/healthcheck.py @@ -1,4 +1,5 @@ import requests + response = requests.post( url='http://0.0.0.0:5000/translate', headers={'Content-Type': 'application/json'}, diff --git a/scripts/install_models.py b/scripts/install_models.py index d844afb..2d3d313 100755 --- a/scripts/install_models.py +++ b/scripts/install_models.py @@ -1,8 +1,10 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) import argparse + from libretranslate.init import check_and_install_models if __name__ == "__main__": diff --git a/scripts/suggestions-to-jsonl.py b/scripts/suggestions-to-jsonl.py index 6cd8e62..7492ac0 100755 --- a/scripts/suggestions-to-jsonl.py +++ b/scripts/suggestions-to-jsonl.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import argparse -import time -import sqlite3 import json +import sqlite3 +import time if __name__ == "__main__": parser = argparse.ArgumentParser(description="Program to generate JSONL files from a LibreTranslate's suggestions.db") diff --git a/scripts/update_locales.py b/scripts/update_locales.py index 1fd2853..93c35fa 100755 --- a/scripts/update_locales.py +++ b/scripts/update_locales.py @@ -1,17 +1,19 @@ #!/usr/bin/env python -import sys import os +import sys + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -import re -import polib import json +import re + +import polib from babel.messages.frontend import main as pybabel -from libretranslate.language import load_languages, improve_translation_formatting -from libretranslate.locales import get_available_locale_codes, swag_eval -from translatehtml import translate_html -from libretranslate.app import get_version, create_app -from libretranslate.main import get_args from flask_swagger import swagger +from libretranslate.app import create_app, get_version +from libretranslate.language import improve_translation_formatting, load_languages +from libretranslate.locales import get_available_locale_codes, swag_eval +from libretranslate.main import get_args +from translatehtml import translate_html # Update strings if __name__ == "__main__": @@ -74,7 +76,7 @@ if __name__ == "__main__": if not os.path.isfile(meta_file): with open(meta_file, 'w') as f: f.write(json.dumps({ - 'name': next((lang.name for lang in languages if lang.code == l)), + 'name': next(lang.name for lang in languages if lang.code == l), 'reviewed': False }, indent=4)) print("Wrote %s" % meta_file) From 1c0fb597fb3cf8e1e249b7fc13047e9c1951a368 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 9 Jul 2023 12:38:03 +0200 Subject: [PATCH 6/8] Fixed some ruff warnings: requests without timeout and naming not complying with PEP --- libretranslate/api_keys.py | 2 +- libretranslate/app.py | 14 +++++++------- libretranslate/detect.py | 8 ++++---- libretranslate/language.py | 8 ++++---- libretranslate/security.py | 6 +++--- pyproject.toml | 15 ++++----------- scripts/healthcheck.py | 3 ++- 7 files changed, 25 insertions(+), 31 deletions(-) diff --git a/libretranslate/api_keys.py b/libretranslate/api_keys.py index 0583531..ea9e603 100644 --- a/libretranslate/api_keys.py +++ b/libretranslate/api_keys.py @@ -87,7 +87,7 @@ class RemoteDatabase: req_limit = self.cache.get(api_key) if req_limit is None: try: - r = requests.post(self.url, data={'api_key': api_key}) + r = requests.post(self.url, data={'api_key': api_key}, timeout=60) res = r.json() except Exception as e: print("Cannot authenticate API key: " + str(e)) diff --git a/libretranslate/app.py b/libretranslate/app.py index 77197f2..39ad1ec 100644 --- a/libretranslate/app.py +++ b/libretranslate/app.py @@ -129,8 +129,8 @@ def create_app(args): from libretranslate.language import load_languages - SWAGGER_URL = args.url_prefix + "/docs" # Swagger UI (w/o trailing '/') - API_URL = args.url_prefix + "/spec" + swagger_url = args.url_prefix + "/docs" # Swagger UI (w/o trailing '/') + api_url = args.url_prefix + "/spec" bp = Blueprint('Main app', __name__) @@ -339,7 +339,7 @@ def create_app(args): get_api_key_link=args.get_api_key_link, web_version=os.environ.get("LT_WEB") is not None, version=get_version(), - swagger_url=SWAGGER_URL, + swagger_url=swagger_url, available_locales=[{'code': l['code'], 'name': _lazy(l['name'])} for l in get_available_locales(not args.debug)], current_locale=get_locale(), alternate_locales=get_alternate_locale_links() @@ -792,7 +792,7 @@ def create_app(args): checked_filepath = security.path_traversal_check(filepath, get_upload_dir()) if os.path.isfile(checked_filepath): filepath = checked_filepath - except security.SuspiciousFileOperation: + except security.SuspiciousFileOperationError: abort(400, description=_("Invalid filename")) return_data = io.BytesIO() @@ -1075,7 +1075,7 @@ def create_app(args): swag["info"]["version"] = get_version() swag["info"]["title"] = "LibreTranslate" - @app.route(API_URL) + @app.route(api_url) @limiter.exempt def spec(): return jsonify(lazy_swag(swag)) @@ -1093,9 +1093,9 @@ def create_app(args): app.jinja_env.globals.update(_e=gettext_escaped, _h=gettext_html) # Call factory function to create our blueprint - swaggerui_blueprint = get_swaggerui_blueprint(SWAGGER_URL, API_URL) + swaggerui_blueprint = get_swaggerui_blueprint(swagger_url, api_url) if args.url_prefix: - app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL) + app.register_blueprint(swaggerui_blueprint, url_prefix=swagger_url) else: app.register_blueprint(swaggerui_blueprint) diff --git a/libretranslate/detect.py b/libretranslate/detect.py index 8717658..b9f6f2e 100644 --- a/libretranslate/detect.py +++ b/libretranslate/detect.py @@ -3,7 +3,7 @@ import pycld2 as cld2 -class UnknownLanguage(Exception): +class UnknownLanguageError(Exception): pass class Language: @@ -57,9 +57,9 @@ class Detector: if not reliable: self.reliable = False reliable, index, top_3_choices = cld2.detect(text, bestEffort=True) - + if not self.quiet and not reliable: - raise UnknownLanguage("Try passing a longer snippet of text") + raise UnknownLanguageError("Try passing a longer snippet of text") self.languages = [Language(x) for x in top_3_choices] self.language = self.languages[0] @@ -69,4 +69,4 @@ class Detector: text = f"Prediction is reliable: {self.reliable}\n" text += "\n".join([f"Language {i+1}: {str(l)}" for i,l in enumerate(self.languages)]) - return text \ No newline at end of file + return text diff --git a/libretranslate/language.py b/libretranslate/language.py index 68e3e48..75fd0a7 100644 --- a/libretranslate/language.py +++ b/libretranslate/language.py @@ -1,7 +1,7 @@ from argostranslate import translate -from libretranslate.detect import Detector, UnknownLanguage +from libretranslate.detect import Detector, UnknownLanguageError __languages = None @@ -29,7 +29,7 @@ def detect_languages(text): for i in range(len(d)): d[i].text_length = len(t) candidates.extend(d) - except UnknownLanguage: + except UnknownLanguageError: pass # total read bytes of the provided text @@ -83,10 +83,10 @@ def improve_translation_formatting(source, translation, improve_punctuation=True if not len(source): return "" - + if not len(translation): return source - + if improve_punctuation: source_last_char = source[len(source) - 1] translation_last_char = translation[len(translation) - 1] diff --git a/libretranslate/security.py b/libretranslate/security.py index 3c4023a..7163a83 100644 --- a/libretranslate/security.py +++ b/libretranslate/security.py @@ -1,7 +1,7 @@ import os -class SuspiciousFileOperation(Exception): +class SuspiciousFileOperationError(Exception): pass @@ -10,7 +10,7 @@ def path_traversal_check(unsafe_path, known_safe_path): unsafe_path = os.path.abspath(unsafe_path) if (os.path.commonprefix([known_safe_path, unsafe_path]) != known_safe_path): - raise SuspiciousFileOperation(f"{unsafe_path} is not safe") + raise SuspiciousFileOperationError(f"{unsafe_path} is not safe") # Passes the check - return unsafe_path \ No newline at end of file + return unsafe_path diff --git a/pyproject.toml b/pyproject.toml index e9eaed8..17b9fb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ keywords = [ "Python", "Translate", "Translation", + "API", ] classifiers = [ "Operating System :: OS Independent", @@ -64,9 +65,8 @@ ltmanage = "libretranslate.manage:manage" test = [ "pytest >=7.2.0", "pytest-cov", - "flake8", + "ruff ==0.0.277", "types-requests", - # "mypy >=1.4.1", ] @@ -86,16 +86,11 @@ features = [ [tool.hatch.envs.default.scripts] dev = "python main.py {args}" -lint = [ - # "flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics", - # "flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics", - "ruff libretranslate scripts", -] fmt = [ "ruff libretranslate scripts --fix", ] test = [ - # "fmt", + "fmt", "pytest {args}", ] cov = [ @@ -124,7 +119,7 @@ addopts = [ ] -# https://github.com/charliermarsh/ruff#supported-rules +# https://beta.ruff.rs/docs/rules [tool.ruff] src = ["libretranslate", "scripts"] target-version = "py38" @@ -155,8 +150,6 @@ select = [ ] ignore = [ - # "E741", # From original flake8 ignore - # "B008", # do not perform function calls in argument defaults (required for FastAPI afaik) "E501", # line too long "A003", # Class attribute is shadowing a python builtin "S101", # Use of `assert` detected diff --git a/scripts/healthcheck.py b/scripts/healthcheck.py index 20eba19..07bcaa4 100644 --- a/scripts/healthcheck.py +++ b/scripts/healthcheck.py @@ -7,6 +7,7 @@ response = requests.post( 'q': 'Hello World!', 'source': 'en', 'target': 'en' - } + }, + timeout=60 ) # if server unavailable then requests with raise exception and healthcheck will fail From a1eb871bb65701ccbf9ff34b1fb07c866de8a5fd Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Mon, 10 Jul 2023 08:11:51 +0200 Subject: [PATCH 7/8] Edit publish workflow to reuse existing test workflow, and use the build package to build the libretranslate package --- .github/workflows/publish-package.yml | 49 ++++++--------------------- .github/workflows/run-tests.yml | 14 +++----- libretranslate/init.py | 6 ++-- pyproject.toml | 5 +-- setup.cfg | 12 ------- 5 files changed, 20 insertions(+), 66 deletions(-) delete mode 100644 setup.cfg diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 5b752ba..aaa2304 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -9,35 +9,8 @@ on: jobs: tests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest flake8 - pip install . - python scripts/compile_locales.py - - - name: Check code style with flake8 (lint) - run: | - # warnings if there are Python syntax errors or undefined names - # (remove --exit-zero to fail when syntax error) - flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Test with pytest - run: pytest + uses: LibreTranslate/LibreTranslate/.github/workflows/run-tests.yml@main + secrets: inherit publish: @@ -52,16 +25,16 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - python setup.py sdist bdist_wheel + pipx install build - - name: Build and publish to PyPI - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - name: Build run: | pip install Babel==2.11.0 python scripts/compile_locales.py - python setup.py sdist bdist_wheel - twine upload dist/* + python -m build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: ${{ secrets.PYPI_USERNAME }} + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a14d001..2703870 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -2,6 +2,7 @@ name: Run tests # Run test at each push to main, if changes to package or tests files on: workflow_dispatch: + workflow_call: pull_request: branches: [ main ] push: @@ -29,18 +30,11 @@ jobs: - name: Install dependencies run: | - pip install ".[test]" - - - name: Check code style with flake8 (lint) - run: | - # warnings if there are Python syntax errors or undefined names - # (remove --exit-zero to fail when syntax error) - flake8 . --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + pipx install hatch + hatch run locales - name: Test with pytest - run: pytest -v + run: hatch run test test_docker_build: diff --git a/libretranslate/init.py b/libretranslate/init.py index 04aa866..eec788d 100644 --- a/libretranslate/init.py +++ b/libretranslate/init.py @@ -45,14 +45,12 @@ def check_and_install_models(force=False, load_only_lang_codes=None): # Download and install all available packages for available_package in available_packages: print( - "Downloading %s (%s) ..." - % (available_package, available_package.package_version) + f"Downloading {available_package} ({available_package.package_version}) ..." ) available_package.install() # reload installed languages libretranslate.language.languages = translate.get_installed_languages() print( - "Loaded support for %s languages (%s models total)!" - % (len(translate.get_installed_languages()), len(available_packages)) + f"Loaded support for {len(translate.get_installed_languages())} languages ({len(available_packages)} models total)!" ) diff --git a/pyproject.toml b/pyproject.toml index 17b9fb1..d7d1252 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,8 +84,10 @@ features = [ "test", ] + [tool.hatch.envs.default.scripts] dev = "python main.py {args}" +locales = "python scripts/compile_locales.py" fmt = [ "ruff libretranslate scripts --fix", ] @@ -145,8 +147,7 @@ select = [ "PLE", # pylint error # "PLR", # pylint refactor # "PLW", # pylint warning - "RUF", # ruff specific - "T", + # "RUF", # ruff specific ] ignore = [ diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e7ca7e4..0000000 --- a/setup.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[flake8] -exclude = .git, - .vscode, - .gitignore, - README.md, - venv, - test, - setup.py, - libretranslate/__init__.py - -max-line-length = 136 -ignore = E741 From 697be38fab5d33ab46f5926ae99547fd362ec26c Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Mon, 10 Jul 2023 08:46:37 +0200 Subject: [PATCH 8/8] Use pip to install build instead of pipx in the publish workflow, and add link to contributing --- .github/workflows/publish-package.yml | 2 +- CONTRIBUTING.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index aaa2304..a68ae51 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -25,7 +25,7 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - pipx install build + pip install build - name: Build run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b04ea3..d5add11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ sudo dnf install cmake ## Getting Started -Install `hatch` to manage the projects dependencies and run dev scripts: +Install [`hatch`](https://hatch.pypa.io) to manage the projects dependencies and run dev scripts: ```bash pipx install hatch