From 25168ba6e18118066ea3b85b4a8807d5b2a91b2a Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 23 Jul 2023 13:24:15 +0200 Subject: [PATCH 1/2] Add pre-commit to automatically run formatting when committing --- .pre-commit-config.yaml | 28 ++++++++++++++++++++++++++++ CONTRIBUTING.md | 10 +++++++++- k8s.yaml | 4 ++-- libretranslate/default_values.py | 2 +- libretranslate/locales.py | 2 +- libretranslate/no_limiter.py | 2 +- libretranslate/scheduler.py | 2 +- libretranslate/secret.py | 2 +- libretranslate/storage.py | 28 ++++++++++++++-------------- pyproject.toml | 7 +++++-- scripts/gunicorn_conf.py | 2 +- scripts/suggestions-to-jsonl.py | 6 +++--- scripts/update_locales.py | 12 ++++++------ wsgi.py | 1 + 14 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..365e151 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +# See https://pre-commit.com for more information +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + name: " 🐘 Check for added large files" + - id: check-toml + name: " ✔️ Check TOML" + - id: check-yaml + name: " ✔️ Check YAML" + args: + - --unsafe + - id: check-json + name: " ✔️ Check JSON" + - id: trailing-whitespace + name: " ✂️ Trim trailing whitespaces" + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.277 + hooks: + - id: ruff + name: " ⚡️ Formatting code with Ruff" + args: + - --fix + +ci: + autofix_commit_msg: 🎨 [pre-commit] Auto format + autoupdate_commit_msg: ⬆ [pre-commit] pre-commit auto update diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd00ac8..15038e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,8 @@ git clone https://github.com/LibreTranslate/LibreTranslate.git cd LibreTranslate ``` +Hatch will automatically install the required dependencies in a virtual environment, and enable [`pre-commit`](https://pre-commit.com/), which will run before each commit to run formatting. + Run in development: ```bash @@ -72,6 +74,12 @@ You can also run the tests on multiple python versions: hatch run all:test ``` +You can clean the virtual environment with: + +```bash +hatch env prune +``` + ## Run with Docker Linux/macOS: `./run.sh [args]` @@ -125,4 +133,4 @@ This occurs when your operating system depends on and manages Python for core fu This prevents pip packages from being installed system-wide. This way, there are no risks of pip packages conflicting between multiple projects or the operating system. References: -* [Python venv documentation](https://docs.python.org/library/venv.html) +* [Python venv documentation](https://docs.python.org/library/venv.html) diff --git a/k8s.yaml b/k8s.yaml index 5760804..06291cd 100644 --- a/k8s.yaml +++ b/k8s.yaml @@ -1,4 +1,4 @@ -# kubernetes deployment template +# kubernetes deployment template # prepare a namespace on your cluster first like libretranslate-prod apiVersion: v1 kind: ConfigMap @@ -41,7 +41,7 @@ spec: valueFrom: configMapKeyRef: name: libretranslate-config - key: ltapikey + key: ltapikey --- apiVersion: v1 kind: Service diff --git a/libretranslate/default_values.py b/libretranslate/default_values.py index 3e14580..d306386 100644 --- a/libretranslate/default_values.py +++ b/libretranslate/default_values.py @@ -170,7 +170,7 @@ _default_options_objects = [ 'name': 'UPDATE_MODELS', 'default_value': False, 'value_type': 'bool' - }, + }, { 'name': 'METRICS', 'default_value': False, diff --git a/libretranslate/locales.py b/libretranslate/locales.py index 9c08d4c..eab1a5f 100644 --- a/libretranslate/locales.py +++ b/libretranslate/locales.py @@ -44,7 +44,7 @@ def get_alternate_locale_links(): tmpl = os.environ.get("LT_LOCALE_LINK_TEMPLATE") if tmpl is None: return [] - + locales = get_available_locale_codes() result = [] for l in locales: diff --git a/libretranslate/no_limiter.py b/libretranslate/no_limiter.py index 2715195..28d895e 100644 --- a/libretranslate/no_limiter.py +++ b/libretranslate/no_limiter.py @@ -8,6 +8,6 @@ class Limiter: return f(*args, **kwargs) return wrapper - + def init_app(self, app): pass diff --git a/libretranslate/scheduler.py b/libretranslate/scheduler.py index de54884..932718e 100644 --- a/libretranslate/scheduler.py +++ b/libretranslate/scheduler.py @@ -18,7 +18,7 @@ def setup(args): if args.api_keys and args.require_api_key_secret: scheduler.add_job(func=rotate_secrets, trigger="interval", minutes=30) - + scheduler.start() # Shut down the scheduler when exiting the app diff --git a/libretranslate/secret.py b/libretranslate/secret.py index 76cf606..8778ff7 100644 --- a/libretranslate/secret.py +++ b/libretranslate/secret.py @@ -12,7 +12,7 @@ def rotate_secrets(): secret_1 = s.get_str("secret_1") s.set_str("secret_0", secret_1) s.set_str("secret_1", generate_secret()) - + def secret_match(secret): s = get_storage() diff --git a/libretranslate/storage.py b/libretranslate/storage.py index 22422f8..46bd9c5 100644 --- a/libretranslate/storage.py +++ b/libretranslate/storage.py @@ -31,22 +31,22 @@ class Storage: raise Exception("not implemented") def dec_hash_int(self, ns, key): raise Exception("not implemented") - + def get_hash_keys(self, ns): raise Exception("not implemented") def del_hash(self, ns, key): raise Exception("not implemented") - + class MemoryStorage(Storage): def __init__(self): self.store = {} - + def exists(self, key): return key in self.store def set_bool(self, key, value): self.store[key] = bool(value) - + def get_bool(self, key): return bool(self.store[key]) @@ -55,10 +55,10 @@ class MemoryStorage(Storage): def get_int(self, key): return int(self.store.get(key, 0)) - + def set_str(self, key, value): self.store[key] = value - + def get_str(self, key): return str(self.store.get(key, "")) @@ -74,7 +74,7 @@ class MemoryStorage(Storage): def inc_hash_int(self, ns, key): if ns not in self.store: self.store[ns] = {} - + if key not in self.store[ns]: self.store[ns][key] = 0 else: @@ -83,7 +83,7 @@ class MemoryStorage(Storage): def dec_hash_int(self, ns, key): if ns not in self.store: self.store[ns] = {} - + if key not in self.store[ns]: self.store[ns][key] = 0 else: @@ -103,13 +103,13 @@ class RedisStorage(Storage): def __init__(self, redis_uri): self.conn = redis.from_url(redis_uri) self.conn.ping() - + def exists(self, key): return bool(self.conn.exists(key)) def set_bool(self, key, value): self.conn.set(key, "1" if value else "0") - + def get_bool(self, key): return bool(self.conn.get(key)) @@ -122,24 +122,24 @@ class RedisStorage(Storage): return 0 else: return v - + def set_str(self, key, value): self.conn.set(key, value) - + def get_str(self, key): v = self.conn.get(key) if v is None: return "" else: return v.decode('utf-8') - + def get_hash_int(self, ns, key): v = self.conn.hget(ns, key) if v is None: return 0 else: return int(v) - + def set_hash_int(self, ns, key, value): self.conn.hset(ns, key, value) diff --git a/pyproject.toml b/pyproject.toml index d7d1252..ad681f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ ltmanage = "libretranslate.manage:manage" test = [ "pytest >=7.2.0", "pytest-cov", - "ruff ==0.0.277", + "pre-commit >=3.0.0", "types-requests", ] @@ -83,13 +83,16 @@ History = "https://github.com/LibreTranslate/LibreTranslate/releases" features = [ "test", ] +post-install-commands = [ + "pre-commit install", +] [tool.hatch.envs.default.scripts] dev = "python main.py {args}" locales = "python scripts/compile_locales.py" fmt = [ - "ruff libretranslate scripts --fix", + "pre-commit run --all --all-files", ] test = [ "fmt", diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index 8c4f709..6909d38 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -36,7 +36,7 @@ def on_starting(server): sys.argv.append(kwargs[k]) args = get_args() - + from libretranslate import flood, scheduler, secret, storage storage.setup(args.shared_storage) scheduler.setup(args) diff --git a/scripts/suggestions-to-jsonl.py b/scripts/suggestions-to-jsonl.py index 7492ac0..c23da91 100755 --- a/scripts/suggestions-to-jsonl.py +++ b/scripts/suggestions-to-jsonl.py @@ -25,19 +25,19 @@ if __name__ == "__main__": con = sqlite3.connect(args.db, check_same_thread=False) cur = con.cursor() - + with open(output_file, 'w', encoding="utf-8") as f: for row in cur.execute('SELECT q, s, source, target FROM suggestions WHERE source != "auto" ORDER BY source'): q, s, source, target = row obj = { 'q': q, - 's': s, + 's': s, 'source': source, 'target': target } json.dump(obj, f, ensure_ascii=False) f.write('\n') - + print("Wrote %s" % output_file) if args.clear: diff --git a/scripts/update_locales.py b/scripts/update_locales.py index 93c35fa..42e5490 100755 --- a/scripts/update_locales.py +++ b/scripts/update_locales.py @@ -54,7 +54,7 @@ if __name__ == "__main__": messagespot = os.path.join(locales_dir, "messages.pot") print("Updating %s" % messagespot) - sys.argv = ["", "extract", "-F", "babel.cfg", "-k", "_e _h", + sys.argv = ["", "extract", "-F", "babel.cfg", "-k", "_e _h", "--copyright-holder", "LibreTranslate Authors", "--project", "LibreTranslate", "--version", get_version(), @@ -80,7 +80,7 @@ if __name__ == "__main__": 'reviewed': False }, indent=4)) print("Wrote %s" % meta_file) - + # Automatically translate strings with libretranslate # when a language model is available and a string is empty @@ -103,10 +103,10 @@ if __name__ == "__main__": print("Translating '%s'" % locale) pofile = polib.pofile(messages_file) c = 0 - + for entry in pofile.untranslated_entries(): text = entry.msgid - + # Extract placeholders placeholders = re.findall(r'%\(?[^\)]*\)?s', text) @@ -126,11 +126,11 @@ if __name__ == "__main__": else: # Meh, append translated += " " + placeholders[p] - + print(entry.msgid, " --> ", translated) entry.msgstr = translated c += 1 - + if c > 0: pofile.save(messages_file) print("Saved %s" % messages_file) diff --git a/wsgi.py b/wsgi.py index 35d9fa8..87413ae 100644 --- a/wsgi.py +++ b/wsgi.py @@ -1,5 +1,6 @@ from libretranslate import main + def app(*args, **kwargs): import sys sys.argv = ['--wsgi'] From f5a6c9679816525ccd82c1d0ee536bdde2ff5d92 Mon Sep 17 00:00:00 2001 From: Vincent Emonet Date: Sun, 23 Jul 2023 17:03:49 +0200 Subject: [PATCH 2/2] add detail to contributing.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15038e4..4bd0221 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ git clone https://github.com/LibreTranslate/LibreTranslate.git cd LibreTranslate ``` -Hatch will automatically install the required dependencies in a virtual environment, and enable [`pre-commit`](https://pre-commit.com/), which will run before each commit to run formatting. +Hatch will automatically install the required dependencies in a virtual environment, and enable [`pre-commit`](https://pre-commit.com/), which will run before each commit to run formatting. You can ignore `pre-commit` checks when committing if necessary: `git commit --no-verify -m "Fix"` Run in development: