From 78713d8017217264bb216bd3ca09eabe5693d47b Mon Sep 17 00:00:00 2001 From: Robert George Date: Fri, 29 Sep 2023 13:43:59 -0700 Subject: [PATCH 1/2] Added option to update models rather than reinstall --- libretranslate/app.py | 3 +-- libretranslate/default_values.py | 5 +++++ libretranslate/init.py | 35 ++++++++++++++++++++++++-------- libretranslate/main.py | 3 +++ scripts/install_models.py | 6 +++++- 5 files changed, 40 insertions(+), 12 deletions(-) diff --git a/libretranslate/app.py b/libretranslate/app.py index 39ad1ec..3af558d 100644 --- a/libretranslate/app.py +++ b/libretranslate/app.py @@ -125,7 +125,7 @@ def get_routes_limits(default_req_limit, daily_req_limit, api_keys_db): def create_app(args): from libretranslate.init import boot - boot(args.load_only, args.update_models) + boot(args.load_only, args.update_models, args.install_models) from libretranslate.language import load_languages @@ -1042,7 +1042,6 @@ def create_app(args): s = request.values.get("s") source_lang = request.values.get("source") target_lang = request.values.get("target") - if not q: abort(400, description=_("Invalid request: missing %(name)s parameter", name='q')) if not s: diff --git a/libretranslate/default_values.py b/libretranslate/default_values.py index d306386..5b526fb 100644 --- a/libretranslate/default_values.py +++ b/libretranslate/default_values.py @@ -171,6 +171,11 @@ _default_options_objects = [ 'default_value': False, 'value_type': 'bool' }, + { + 'name': 'INSTALL_MODELS', + 'default_value': False, + 'value_type': 'bool' + }, { 'name': 'METRICS', 'default_value': False, diff --git a/libretranslate/init.py b/libretranslate/init.py index eec788d..46a8b74 100644 --- a/libretranslate/init.py +++ b/libretranslate/init.py @@ -4,23 +4,26 @@ from argostranslate import package, translate import libretranslate.language -def boot(load_only=None, update_models=False): +def boot(load_only=None, update_models=False, install_models=False): try: - check_and_install_models(force=update_models, load_only_lang_codes=load_only) + if update_models: + check_and_install_models(load_only_lang_codes=load_only, update=update_models) + else: + check_and_install_models(force=install_models, load_only_lang_codes=load_only) except Exception as e: print("Cannot update models (normal if you're offline): %s" % str(e)) -def check_and_install_models(force=False, load_only_lang_codes=None): - if len(package.get_installed_packages()) < 2 or force: +def check_and_install_models(force=False, load_only_lang_codes=None,update=False): + if len(package.get_installed_packages()) < 2 or force or update: # Update package definitions from remote print("Updating language models") package.update_package_index() # Load available packages from local package index available_packages = package.get_available_packages() + installed_packages = package.get_installed_packages() print("Found %s models" % len(available_packages)) - if load_only_lang_codes is not None: # load_only_lang_codes: List[str] (codes) # Ensure the user does not use any unavailable language code. @@ -44,10 +47,24 @@ def check_and_install_models(force=False, load_only_lang_codes=None): # Download and install all available packages for available_package in available_packages: - print( - f"Downloading {available_package} ({available_package.package_version}) ..." - ) - available_package.install() + update = False + if not force: + for pack in installed_packages: + if ( + pack.from_code == available_package.from_code + and pack.to_code == available_package.to_code + ): + update = True + if pack.package_version < available_package.package_version: + print( + f"Updating {available_package} ({pack.package_version}->{available_package.package_version}) ..." + ) + pack.update() + if not update: + print( + f"Downloading {available_package} ({available_package.package_version}) ..." + ) + available_package.install() # reload installed languages libretranslate.language.languages = translate.get_installed_languages() diff --git a/libretranslate/main.py b/libretranslate/main.py index 57dd723..41035fc 100644 --- a/libretranslate/main.py +++ b/libretranslate/main.py @@ -160,6 +160,9 @@ def get_args(): parser.add_argument( "--update-models", default=DEFARGS['UPDATE_MODELS'], action="store_true", help="Update language models at startup" ) + parser.add_argument( + "--install-models", default=DEFARGS['INSTALL_MODELS'], action="store_true", help="Install language models at startup" + ) parser.add_argument( "--metrics", default=DEFARGS['METRICS'], diff --git a/scripts/install_models.py b/scripts/install_models.py index 2d3d313..f60d07c 100755 --- a/scripts/install_models.py +++ b/scripts/install_models.py @@ -10,8 +10,12 @@ from libretranslate.init import check_and_install_models if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--load_only_lang_codes", type=str, default="") + parser.add_argument("--update", action='store_true') args = parser.parse_args() lang_codes = args.load_only_lang_codes.split(",") if len(lang_codes) == 0 or lang_codes[0] == '': lang_codes = None - check_and_install_models(force=True, load_only_lang_codes=lang_codes) + if args.update: + check_and_install_models(update=True, load_only_lang_codes=lang_codes) + else: + check_and_install_models(force=True, load_only_lang_codes=lang_codes) From 94f63cedda50521620ba56d19f22fde66cd590c6 Mon Sep 17 00:00:00 2001 From: Robert George Date: Fri, 29 Sep 2023 15:31:33 -0700 Subject: [PATCH 2/2] Use package.version for model version comparison Change `--install-models` to `--force-update-models` --- libretranslate/app.py | 2 +- libretranslate/default_values.py | 2 +- libretranslate/init.py | 3 ++- libretranslate/main.py | 2 +- pyproject.toml | 1 + 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libretranslate/app.py b/libretranslate/app.py index 3af558d..0a849c6 100644 --- a/libretranslate/app.py +++ b/libretranslate/app.py @@ -125,7 +125,7 @@ def get_routes_limits(default_req_limit, daily_req_limit, api_keys_db): def create_app(args): from libretranslate.init import boot - boot(args.load_only, args.update_models, args.install_models) + boot(args.load_only, args.update_models, args.force_update_models) from libretranslate.language import load_languages diff --git a/libretranslate/default_values.py b/libretranslate/default_values.py index 5b526fb..df058a3 100644 --- a/libretranslate/default_values.py +++ b/libretranslate/default_values.py @@ -172,7 +172,7 @@ _default_options_objects = [ 'value_type': 'bool' }, { - 'name': 'INSTALL_MODELS', + 'name': 'FORCE_UPDATE_MODELS', 'default_value': False, 'value_type': 'bool' }, diff --git a/libretranslate/init.py b/libretranslate/init.py index 46a8b74..098cb20 100644 --- a/libretranslate/init.py +++ b/libretranslate/init.py @@ -1,5 +1,6 @@ from argostranslate import package, translate +from packaging import version import libretranslate.language @@ -55,7 +56,7 @@ def check_and_install_models(force=False, load_only_lang_codes=None,update=False and pack.to_code == available_package.to_code ): update = True - if pack.package_version < available_package.package_version: + if version.parse(pack.package_version) < version.parse(available_package.package_version): print( f"Updating {available_package} ({pack.package_version}->{available_package.package_version}) ..." ) diff --git a/libretranslate/main.py b/libretranslate/main.py index 41035fc..84589d3 100644 --- a/libretranslate/main.py +++ b/libretranslate/main.py @@ -161,7 +161,7 @@ def get_args(): "--update-models", default=DEFARGS['UPDATE_MODELS'], action="store_true", help="Update language models at startup" ) parser.add_argument( - "--install-models", default=DEFARGS['INSTALL_MODELS'], action="store_true", help="Install language models at startup" + "--force-update-models", default=DEFARGS['FORCE_UPDATE_MODELS'], action="store_true", help="Install/Reinstall language models at startup" ) parser.add_argument( "--metrics", diff --git a/pyproject.toml b/pyproject.toml index a766252..445091b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dependencies = [ "redis ==4.3.4", "prometheus-client ==0.15.0", "polib ==1.1.1", + "packaging ==23.1" ] [project.scripts]