1
0
mirror of https://github.com/LibreTranslate/LibreTranslate.git synced 2024-11-15 04:00:11 +01:00

Merge pull request #503 from rrgeorge/add_update_option

Add option to update models rather than reinstall
This commit is contained in:
Piero Toffanin 2023-09-29 21:34:44 -04:00 committed by GitHub
commit 33f12a8ebb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 42 additions and 11 deletions

View File

@ -125,7 +125,7 @@ def get_routes_limits(default_req_limit, daily_req_limit, api_keys_db):
def create_app(args):
from libretranslate.init import boot
boot(args.load_only, args.update_models)
boot(args.load_only, args.update_models, args.force_update_models)
from libretranslate.language import load_languages

View File

@ -171,6 +171,11 @@ _default_options_objects = [
'default_value': False,
'value_type': 'bool'
},
{
'name': 'FORCE_UPDATE_MODELS',
'default_value': False,
'value_type': 'bool'
},
{
'name': 'METRICS',
'default_value': False,

View File

@ -1,26 +1,30 @@
from argostranslate import package, translate
from packaging import version
import libretranslate.language
def boot(load_only=None, update_models=False):
def boot(load_only=None, update_models=False, install_models=False):
try:
check_and_install_models(force=update_models, load_only_lang_codes=load_only)
if update_models:
check_and_install_models(load_only_lang_codes=load_only, update=update_models)
else:
check_and_install_models(force=install_models, load_only_lang_codes=load_only)
except Exception as e:
print("Cannot update models (normal if you're offline): %s" % str(e))
def check_and_install_models(force=False, load_only_lang_codes=None):
if len(package.get_installed_packages()) < 2 or force:
def check_and_install_models(force=False, load_only_lang_codes=None,update=False):
if len(package.get_installed_packages()) < 2 or force or update:
# Update package definitions from remote
print("Updating language models")
package.update_package_index()
# Load available packages from local package index
available_packages = package.get_available_packages()
installed_packages = package.get_installed_packages()
print("Found %s models" % len(available_packages))
if load_only_lang_codes is not None:
# load_only_lang_codes: List[str] (codes)
# Ensure the user does not use any unavailable language code.
@ -44,10 +48,24 @@ def check_and_install_models(force=False, load_only_lang_codes=None):
# Download and install all available packages
for available_package in available_packages:
print(
f"Downloading {available_package} ({available_package.package_version}) ..."
)
available_package.install()
update = False
if not force:
for pack in installed_packages:
if (
pack.from_code == available_package.from_code
and pack.to_code == available_package.to_code
):
update = True
if version.parse(pack.package_version) < version.parse(available_package.package_version):
print(
f"Updating {available_package} ({pack.package_version}->{available_package.package_version}) ..."
)
pack.update()
if not update:
print(
f"Downloading {available_package} ({available_package.package_version}) ..."
)
available_package.install()
# reload installed languages
libretranslate.language.languages = translate.get_installed_languages()

View File

@ -160,6 +160,9 @@ def get_args():
parser.add_argument(
"--update-models", default=DEFARGS['UPDATE_MODELS'], action="store_true", help="Update language models at startup"
)
parser.add_argument(
"--force-update-models", default=DEFARGS['FORCE_UPDATE_MODELS'], action="store_true", help="Install/Reinstall language models at startup"
)
parser.add_argument(
"--metrics",
default=DEFARGS['METRICS'],

View File

@ -54,6 +54,7 @@ dependencies = [
"redis ==4.3.4",
"prometheus-client ==0.15.0",
"polib ==1.1.1",
"packaging ==23.1"
]
[project.scripts]

View File

@ -10,8 +10,12 @@ from libretranslate.init import check_and_install_models
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--load_only_lang_codes", type=str, default="")
parser.add_argument("--update", action='store_true')
args = parser.parse_args()
lang_codes = args.load_only_lang_codes.split(",")
if len(lang_codes) == 0 or lang_codes[0] == '':
lang_codes = None
check_and_install_models(force=True, load_only_lang_codes=lang_codes)
if args.update:
check_and_install_models(update=True, load_only_lang_codes=lang_codes)
else:
check_and_install_models(force=True, load_only_lang_codes=lang_codes)