From 3cbbd8ae16c5426534de5b14ea24b7b426f55f9c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Jan 2023 15:36:26 -0500 Subject: [PATCH] Auto self-translation --- compile_locales.py | 16 ++++ compile_translations.py | 16 ---- libretranslate/app.py | 5 +- libretranslate/locales.py | 9 ++ .../{translations => locales}/.gitignore | 0 .../it/LC_MESSAGES/messages.po | 17 ++-- libretranslate/templates/app.js.template | 4 +- requirements.txt | 1 + update_locales.py | 96 +++++++++++++++++++ update_translations.py | 31 ------ 10 files changed, 135 insertions(+), 60 deletions(-) create mode 100755 compile_locales.py delete mode 100755 compile_translations.py create mode 100644 libretranslate/locales.py rename libretranslate/{translations => locales}/.gitignore (100%) rename libretranslate/{translations => locales}/it/LC_MESSAGES/messages.po (82%) create mode 100755 update_locales.py delete mode 100755 update_translations.py diff --git a/compile_locales.py b/compile_locales.py new file mode 100755 index 0000000..300bbb0 --- /dev/null +++ b/compile_locales.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +import sys +import os +from babel.messages.frontend import main as pybabel + +if __name__ == "__main__": + locales_dir = os.path.join("libretranslate", "locales") + if not os.path.isdir(locales_dir): + os.makedirs(locales_dir) + + print("Compiling locales") + sys.argv = ["", "compile", "-d", locales_dir] + pybabel() + + + diff --git a/compile_translations.py b/compile_translations.py deleted file mode 100755 index 55076d3..0000000 --- a/compile_translations.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import sys -import os -from babel.messages.frontend import main as pybabel - -if __name__ == "__main__": - translations_dir = os.path.join("libretranslate", "translations") - if not os.path.isdir(translations_dir): - os.makedirs(translations_dir) - - print("Compiling translations") - sys.argv = ["", "compile", "-d", translations_dir] - pybabel() - - - diff --git a/libretranslate/app.py b/libretranslate/app.py index 52e818f..964a15d 100644 --- a/libretranslate/app.py +++ b/libretranslate/app.py @@ -19,6 +19,7 @@ from flask_babel import Babel, gettext as _ from libretranslate import flood, remove_translated_files, security from libretranslate.language import detect_languages, improve_translation_formatting +from libretranslate.locales import get_available_locales from .api_keys import Database, RemoteDatabase from .suggestions import Database as SuggestionsDatabase @@ -1020,11 +1021,11 @@ def create_app(args): @babel.localeselector def get_locale(): # TODO: populate from available locales - return request.accept_languages.best_match(['en', 'it']) + return request.accept_languages.best_match(get_available_locales()) def gettext_escaped(*args, **kwargs): return _(*args, **kwargs).replace("'", "\\'") - app.jinja_env.globals.update(_e=gettext_escaped) + app.jinja_env.globals.update(N_=gettext_escaped) # Call factory function to create our blueprint swaggerui_blueprint = get_swaggerui_blueprint(SWAGGER_URL, API_URL) diff --git a/libretranslate/locales.py b/libretranslate/locales.py new file mode 100644 index 0000000..b3deb4c --- /dev/null +++ b/libretranslate/locales.py @@ -0,0 +1,9 @@ +import os +from functools import cache + +@cache +def get_available_locales(): + locales_dir = os.path.join(os.path.dirname(__file__), 'locales') + dirs = [os.path.join(locales_dir, d) for d in os.listdir(locales_dir)] + + return ['en'] + [os.path.basename(d) for d in dirs if os.path.isdir(os.path.join(d, 'LC_MESSAGES'))] \ No newline at end of file diff --git a/libretranslate/translations/.gitignore b/libretranslate/locales/.gitignore similarity index 100% rename from libretranslate/translations/.gitignore rename to libretranslate/locales/.gitignore diff --git a/libretranslate/translations/it/LC_MESSAGES/messages.po b/libretranslate/locales/it/LC_MESSAGES/messages.po similarity index 82% rename from libretranslate/translations/it/LC_MESSAGES/messages.po rename to libretranslate/locales/it/LC_MESSAGES/messages.po index 9c65540..092dcc5 100644 --- a/libretranslate/translations/it/LC_MESSAGES/messages.po +++ b/libretranslate/locales/it/LC_MESSAGES/messages.po @@ -7,24 +7,24 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2023-01-04 12:27-0500\n" +"POT-Creation-Date: 2023-01-04 15:34-0500\n" "PO-Revision-Date: 2023-01-04 12:27-0500\n" "Last-Translator: FULL NAME \n" -"Language: it\n" "Language-Team: it \n" -"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"Language: it\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" "Generated-By: Babel 2.11.0\n" -#: libretranslate/app.py:57 +#: libretranslate/app.py:58 msgid "Invalid JSON format" -msgstr "" +msgstr "Formato JSON non valido" -#: libretranslate/app.py:125 +#: libretranslate/app.py:126 msgid "Auto Detect" -msgstr "" +msgstr "Rilevamento automatico" #: libretranslate/templates/app.js.template:31 msgid "Copy text" @@ -33,9 +33,8 @@ msgstr "Copia testo" #: libretranslate/templates/app.js.template:72 #, python-format msgid "Cannot load %(url)s" -msgstr "Impossibile caricare' %(url)s" +msgstr "Non riesco a caricare %(url)s" #: libretranslate/templates/index.html:6 msgid "Free and Open Source Machine Translation API" msgstr "API di traduzione automatica open source" - diff --git a/libretranslate/templates/app.js.template b/libretranslate/templates/app.js.template index 57dde04..1aae136 100644 --- a/libretranslate/templates/app.js.template +++ b/libretranslate/templates/app.js.template @@ -28,7 +28,7 @@ document.addEventListener('DOMContentLoaded', function(){ detectedLangText: "", - copyTextLabel: '{{ _e("Copy text") }}', + copyTextLabel: '{{ N_("Copy text") }}', suggestions: false, isSuggesting: false, @@ -69,7 +69,7 @@ document.addEventListener('DOMContentLoaded', function(){ } } } else { - self.error = '{{ _e("Cannot load %(url)s", url="/frontend/settings") }}'; + self.error = '{{ N_("Cannot load %(url)s", url="/frontend/settings") }}'; self.loading = false; } }; diff --git a/requirements.txt b/requirements.txt index f81df00..5f21574 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ Werkzeug==2.2.2 requests==2.28.1 redis==4.3.4 prometheus-client==0.15.0 +polib==1.1.1 diff --git a/update_locales.py b/update_locales.py new file mode 100755 index 0000000..c8f6ebd --- /dev/null +++ b/update_locales.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +import sys +import os +import re +import polib +from babel.messages.frontend import main as pybabel +from libretranslate.language import load_languages, improve_translation_formatting +from libretranslate.locales import get_available_locales +from translatehtml import translate_html + +# Update strings +if __name__ == "__main__": + locales_dir = os.path.join("libretranslate", "locales") + if not os.path.isdir(locales_dir): + os.makedirs(locales_dir) + + messagespot = os.path.join(locales_dir, "messages.pot") + print("Updating %s" % messagespot) + sys.argv = ["", "extract", "-F", "babel.cfg", "-o", messagespot, "libretranslate"] + pybabel() + + # Load list of languages + print("Loading languages") + languages = load_languages() + en_lang = next((l for l in languages if l.code == 'en'), None) + if en_lang is None: + print("Error: English model not found. You need it to run this script.") + exit(1) + + lang_codes = [l.code for l in languages if l != "en"] + lang_codes = ["it"] # TODO REMOVE + + # Init/update + for l in lang_codes: + cmd = "init" + if os.path.isdir(os.path.join(locales_dir, l)): + cmd = "update" + + sys.argv = ["", cmd, "-i", messagespot, "-d", locales_dir, "-l", l] + pybabel() + + # Automatically translate strings with libretranslate + # when a language model is available and a string is empty + + locales = get_available_locales() + for locale in locales: + if locale == 'en': + continue + + tgt_lang = next((l for l in languages if l.code == locale), None) + + if tgt_lang is None: + # We cannot translate + continue + + translator = en_lang.get_translation(tgt_lang) + + messages_file = os.path.join(locales_dir, locale, "LC_MESSAGES", 'messages.po') + if os.path.isfile(messages_file): + print("Translating '%s'" % locale) + pofile = polib.pofile(messages_file) + c = 0 + + for entry in pofile.untranslated_entries(): + text = entry.msgid + + # Extract placeholders + placeholders = re.findall(r'%\(?.*?\)?s', text) + + for p in range(0, len(placeholders)): + text = text.replace(placeholders[p], "%s" % p) + + if len(placeholders) > 0: + translated = str(translate_html(translator, text)) + else: + translated = improve_translation_formatting(text, translator.translate(text)) + + # Restore placeholders + for p in range(0, len(placeholders)): + tag = "%s" % p + if tag in translated: + translated = translated.replace(tag, placeholders[p]) + else: + # Meh, append + translated += " " + placeholders[p] + + print(entry.msgid, " --> ", translated) + entry.msgstr = translated + c += 1 + + if c > 0: + pofile.save(messages_file) + print("Saved %s" % messages_file) + + + diff --git a/update_translations.py b/update_translations.py deleted file mode 100755 index ecc22d9..0000000 --- a/update_translations.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -import sys -import os -from babel.messages.frontend import main as pybabel -from libretranslate.language import load_languages - -# Update strings -if __name__ == "__main__": - translations_dir = os.path.join("libretranslate", "translations") - if not os.path.isdir(translations_dir): - os.makedirs(translations_dir) - - messagespot = os.path.join(translations_dir, "messages.pot") - print("Updating %s" % messagespot) - sys.argv = ["", "extract", "-F", "babel.cfg", "-o", messagespot, "libretranslate"] - pybabel() - - # Load list of languages - print("Loading languages") - languages = [l.code for l in load_languages() if l != "en"] - print(languages) - languages = ["it"] - - for l in languages: - cmd = "init" - if os.path.isdir(os.path.join(translations_dir, l)): - cmd = "update" - - sys.argv = ["", cmd, "-i", messagespot, "-d", translations_dir, "-l", l] - pybabel() -