From b4b81a5e1a74f03926e3c2e6f8c5fade99f7eabb Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 27 Nov 2020 19:32:45 +0100 Subject: [PATCH] [enh] settings.yml: add use_default_settings option (2nd version) --- Makefile | 17 +-- docs/admin/settings.rst | 75 ++++------ searx/__init__.py | 4 +- searx/settings.py | 91 ------------ searx/settings_loader.py | 129 ++++++++++++++++++ tests/unit/settings/empty_settings.yml | 0 tests/unit/settings/syntaxerror_settings.yml | 2 + tests/unit/settings/user_settings.yml | 111 +++++++++++++++ .../unit/settings/user_settings_keep_only.yml | 14 ++ tests/unit/settings/user_settings_remove.yml | 10 ++ tests/unit/settings/user_settings_remove2.yml | 15 ++ tests/unit/settings/user_settings_simple.yml | 6 + tests/unit/test_settings_loader.py | 122 +++++++++++++++++ utils/update_user_settings.py | 98 ------------- 14 files changed, 441 insertions(+), 253 deletions(-) delete mode 100644 searx/settings.py create mode 100644 searx/settings_loader.py create mode 100644 tests/unit/settings/empty_settings.yml create mode 100644 tests/unit/settings/syntaxerror_settings.yml create mode 100644 tests/unit/settings/user_settings.yml create mode 100644 tests/unit/settings/user_settings_keep_only.yml create mode 100644 tests/unit/settings/user_settings_remove.yml create mode 100644 tests/unit/settings/user_settings_remove2.yml create mode 100644 tests/unit/settings/user_settings_simple.yml create mode 100644 tests/unit/test_settings_loader.py delete mode 100644 utils/update_user_settings.py diff --git a/Makefile b/Makefile index 94309cefe..29faaeefa 100644 --- a/Makefile +++ b/Makefile @@ -266,19 +266,4 @@ test.clean: travis.codecov: $(Q)$(PY_ENV_BIN)/python -m pip install codecov - -# user-settings -# ------------- - -PHONY += user-settings.create user-settings.update - -user-settings.update: pyenvinstall - $(Q)$(PY_ENV_ACT); pip install ruamel.yaml - $(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH} - -user-settings.update.engines: pyenvinstall - $(Q)$(PY_ENV_ACT); pip install ruamel.yaml - $(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH} - - -.PHONY: $(PHONY) \ No newline at end of file +.PHONY: $(PHONY) diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst index cd944cc4c..532b99752 100644 --- a/docs/admin/settings.rst +++ b/docs/admin/settings.rst @@ -235,68 +235,51 @@ In the following example, the actual settings are the default settings defined i .. code-block:: yaml - use_default_settings: true + use_default_settings: True server: secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" - server: bind_address: "0.0.0.0" -With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section: +With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``. -* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example). -* If the ``engines`` section is defined then: - - * searx loads only the engines declare in the user setttings. - * searx merges the configuration according to the engine name. - -In the following example, only three engines are available. Each engine configuration is merged with the default configuration. +In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token`: .. code-block:: yaml - use_default_settings: true - server: - secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" - engines: - - name: wikipedia - - name: wikidata - - name: ddg definitions - -Another example where four engines are available. The arch linux wiki engine has a :ref:`token`. - -.. code-block:: yaml - - use_default_settings: true + use_default_settings: True server: secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" engines: - name: arch linux wiki tokens: ['$ecretValue'] - - name: wikipedia - - name: wikidata - - name: ddg definitions -automatic update ----------------- +It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine: -The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case): +.. code-block:: yaml -.. code-block:: sh + use_default_settings: + engines: + remove: + - google + server: + secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" + engines: + - name: arch linux wiki + tokens: ['$ecretValue'] - make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update +As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines: -Set ``SEARX_SETTINGS_PATH`` to your user settings path. +.. code-block:: yaml -As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated. -The following command creates or updates the user settings including the ``engines`` section: - -.. code-block:: sh - - make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines - -After that ``/etc/searx/settings.yml`` - -* has a ``secret key`` -* has a ``engine`` section if it is not already the case, moreover the command: - - * has deleted engines that do not exist in the default settings. - * has added engines that exist in the default settings but are not declare in the user settings. + use_default_settings: + engines: + keep_only: + - google + - duckduckgo + server: + secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA" + engines: + - name: google + tokens: ['$ecretValue'] + - name: duckduckgo + tokens: ['$ecretValue'] diff --git a/searx/__init__.py b/searx/__init__.py index 214e554d4..9bbc7c8c3 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -16,7 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. ''' import logging -import searx.settings +import searx.settings_loader from os import environ from os.path import realpath, dirname, join, abspath, isfile @@ -24,7 +24,7 @@ from os.path import realpath, dirname, join, abspath, isfile searx_dir = abspath(dirname(__file__)) engine_dir = dirname(realpath(__file__)) static_path = abspath(join(dirname(__file__), 'static')) -settings, settings_load_message = searx.settings.load_settings() +settings, settings_load_message = searx.settings_loader.load_settings() if settings['ui']['static_path']: static_path = settings['ui']['static_path'] diff --git a/searx/settings.py b/searx/settings.py deleted file mode 100644 index cdddff589..000000000 --- a/searx/settings.py +++ /dev/null @@ -1,91 +0,0 @@ -import collections.abc - -import yaml -from searx.exceptions import SearxSettingsException -from os import environ -from os.path import dirname, join, abspath, isfile - - -searx_dir = abspath(dirname(__file__)) - - -def check_settings_yml(file_name): - if isfile(file_name): - return file_name - else: - return None - - -def load_yaml(file_name): - try: - with open(file_name, 'r', encoding='utf-8') as settings_yaml: - settings = yaml.safe_load(settings_yaml) - if not isinstance(settings, dict) or len(settings) == 0: - raise SearxSettingsException('Empty file', file_name) - return settings - except IOError as e: - raise SearxSettingsException(e, file_name) - except yaml.YAMLError as e: - raise SearxSettingsException(e, file_name) - - -def get_default_settings_path(): - return check_settings_yml(join(searx_dir, 'settings.yml')) - - -def get_user_settings_path(): - # find location of settings.yml - if 'SEARX_SETTINGS_PATH' in environ: - # if possible set path to settings using the - # enviroment variable SEARX_SETTINGS_PATH - return check_settings_yml(environ['SEARX_SETTINGS_PATH']) - else: - # if not, get it from searx code base or last solution from /etc/searx - return check_settings_yml('/etc/searx/settings.yml') - - -def update_dict(d, u): - for k, v in u.items(): - if isinstance(v, collections.abc.Mapping): - d[k] = update_dict(d.get(k, {}), v) - else: - d[k] = v - return d - - -def update_settings(default_settings, user_settings): - for k, v in user_settings.items(): - if k == 'use_default_settings': - continue - elif k == 'engines': - default_engines = default_settings[k] - default_engines_dict = dict((definition['name'], definition) for definition in default_engines) - default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition) - for definition in v] - else: - update_dict(default_settings[k], v) - - return default_settings - - -def load_settings(load_user_setttings=True): - default_settings_path = get_default_settings_path() - user_settings_path = get_user_settings_path() - if user_settings_path is None or not load_user_setttings: - # no user settings - return (load_yaml(default_settings_path), - 'load the default settings from {}'.format(default_settings_path)) - - # user settings - user_settings = load_yaml(user_settings_path) - if user_settings.get('use_default_settings'): - # the user settings are merged with the default configuration - default_settings = load_yaml(default_settings_path) - update_settings(default_settings, user_settings) - return (default_settings, - 'merge the default settings ( {} ) and the user setttings ( {} )' - .format(default_settings_path, user_settings_path)) - - # the user settings, fully replace the default configuration - return (user_settings, - 'load the user settings from {}'.format(user_settings_path)) diff --git a/searx/settings_loader.py b/searx/settings_loader.py new file mode 100644 index 000000000..172069bd5 --- /dev/null +++ b/searx/settings_loader.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +from os import environ +from os.path import dirname, join, abspath, isfile +from collections.abc import Mapping +from itertools import filterfalse + +import yaml + +from searx.exceptions import SearxSettingsException + + +searx_dir = abspath(dirname(__file__)) + + +def check_settings_yml(file_name): + if isfile(file_name): + return file_name + return None + + +def load_yaml(file_name): + try: + with open(file_name, 'r', encoding='utf-8') as settings_yaml: + return yaml.safe_load(settings_yaml) + except IOError as e: + raise SearxSettingsException(e, file_name) + except yaml.YAMLError as e: + raise SearxSettingsException(e, file_name) + + +def get_default_settings_path(): + return check_settings_yml(join(searx_dir, 'settings.yml')) + + +def get_user_settings_path(): + # find location of settings.yml + if 'SEARX_SETTINGS_PATH' in environ: + # if possible set path to settings using the + # enviroment variable SEARX_SETTINGS_PATH + return check_settings_yml(environ['SEARX_SETTINGS_PATH']) + + # if not, get it from searx code base or last solution from /etc/searx + return check_settings_yml('/etc/searx/settings.yml') + + +def update_dict(default_dict, user_dict): + for k, v in user_dict.items(): + if isinstance(v, Mapping): + default_dict[k] = update_dict(default_dict.get(k, {}), v) + else: + default_dict[k] = v + return default_dict + + +def update_settings(default_settings, user_settings): + # merge everything except the engines + for k, v in user_settings.items(): + if k not in ('use_default_settings', 'engines'): + update_dict(default_settings[k], v) + + # parse the engines + remove_engines = None + keep_only_engines = None + use_default_settings = user_settings.get('use_default_settings') + if isinstance(use_default_settings, dict): + remove_engines = use_default_settings.get('engines', {}).get('remove') + keep_only_engines = use_default_settings.get('engines', {}).get('keep_only') + + if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None: + engines = default_settings['engines'] + + # parse "use_default_settings.engines.remove" + if remove_engines is not None: + engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines)) + + # parse "use_default_settings.engines.keep_only" + if keep_only_engines is not None: + engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines)) + + # parse "engines" + user_engines = user_settings.get('engines') + if user_engines: + engines_dict = dict((definition['name'], definition) for definition in engines) + for user_engine in user_engines: + default_engine = engines_dict.get(user_engine['name']) + if default_engine: + update_dict(default_engine, user_engine) + else: + engines.append(user_engine) + + # store the result + default_settings['engines'] = engines + + return default_settings + + +def is_use_default_settings(user_settings): + use_default_settings = user_settings.get('use_default_settings') + if use_default_settings is True: + return True + if isinstance(use_default_settings, dict): + return True + if use_default_settings is False or use_default_settings is None: + return False + raise ValueError('Invalid value for use_default_settings') + + +def load_settings(load_user_setttings=True): + default_settings_path = get_default_settings_path() + user_settings_path = get_user_settings_path() + if user_settings_path is None or not load_user_setttings: + # no user settings + return (load_yaml(default_settings_path), + 'load the default settings from {}'.format(default_settings_path)) + + # user settings + user_settings = load_yaml(user_settings_path) + if is_use_default_settings(user_settings): + # the user settings are merged with the default configuration + default_settings = load_yaml(default_settings_path) + update_settings(default_settings, user_settings) + return (default_settings, + 'merge the default settings ( {} ) and the user setttings ( {} )' + .format(default_settings_path, user_settings_path)) + + # the user settings, fully replace the default configuration + return (user_settings, + 'load the user settings from {}'.format(user_settings_path)) diff --git a/tests/unit/settings/empty_settings.yml b/tests/unit/settings/empty_settings.yml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/settings/syntaxerror_settings.yml b/tests/unit/settings/syntaxerror_settings.yml new file mode 100644 index 000000000..6d3b0f9a0 --- /dev/null +++ b/tests/unit/settings/syntaxerror_settings.yml @@ -0,0 +1,2 @@ +Test: + ********** diff --git a/tests/unit/settings/user_settings.yml b/tests/unit/settings/user_settings.yml new file mode 100644 index 000000000..f5b6c7173 --- /dev/null +++ b/tests/unit/settings/user_settings.yml @@ -0,0 +1,111 @@ +general: + debug : False + instance_name : "searx" + +search: + safe_search : 0 + autocomplete : "" + default_lang : "" + ban_time_on_fail : 5 + max_ban_time_on_fail : 120 + +server: + port : 9000 + bind_address : "0.0.0.0" + secret_key : "user_settings_secret" + base_url : False + image_proxy : False + http_protocol_version : "1.0" + method: "POST" + default_http_headers: + X-Content-Type-Options : nosniff + X-XSS-Protection : 1; mode=block + X-Download-Options : noopen + X-Robots-Tag : noindex, nofollow + Referrer-Policy : no-referrer + +ui: + static_path : "" + templates_path : "" + default_theme : oscar + default_locale : "" + theme_args : + oscar_style : logicodev + +engines: + - name : wikidata + engine : wikidata + shortcut : wd + timeout : 3.0 + weight : 2 + + - name : wikibooks + engine : mediawiki + shortcut : wb + categories : general + base_url : "https://{language}.wikibooks.org/" + number_of_results : 5 + search_type : text + + - name : wikinews + engine : mediawiki + shortcut : wn + categories : news + base_url : "https://{language}.wikinews.org/" + number_of_results : 5 + search_type : text + + - name : wikiquote + engine : mediawiki + shortcut : wq + categories : general + base_url : "https://{language}.wikiquote.org/" + number_of_results : 5 + search_type : text + +locales: + en : English + ar : العَرَبِيَّة (Arabic) + bg : Български (Bulgarian) + bo : བོད་སྐད་ (Tibetian) + ca : Català (Catalan) + cs : Čeština (Czech) + cy : Cymraeg (Welsh) + da : Dansk (Danish) + de : Deutsch (German) + el_GR : Ελληνικά (Greek_Greece) + eo : Esperanto (Esperanto) + es : Español (Spanish) + et : Eesti (Estonian) + eu : Euskara (Basque) + fa_IR : (fārsī) فارسى (Persian) + fi : Suomi (Finnish) + fil : Wikang Filipino (Filipino) + fr : Français (French) + gl : Galego (Galician) + he : עברית (Hebrew) + hr : Hrvatski (Croatian) + hu : Magyar (Hungarian) + ia : Interlingua (Interlingua) + it : Italiano (Italian) + ja : 日本語 (Japanese) + lt : Lietuvių (Lithuanian) + nl : Nederlands (Dutch) + nl_BE : Vlaams (Dutch_Belgium) + oc : Lenga D'òc (Occitan) + pl : Polski (Polish) + pt : Português (Portuguese) + pt_BR : Português (Portuguese_Brazil) + ro : Română (Romanian) + ru : Русский (Russian) + sk : Slovenčina (Slovak) + sl : Slovenski (Slovene) + sr : српски (Serbian) + sv : Svenska (Swedish) + te : తెలుగు (telugu) + ta : தமிழ் (Tamil) + tr : Türkçe (Turkish) + uk : українська мова (Ukrainian) + vi : tiếng việt (Vietnamese) + zh : 中文 (Chinese) + zh_TW : 國語 (Taiwanese Mandarin) diff --git a/tests/unit/settings/user_settings_keep_only.yml b/tests/unit/settings/user_settings_keep_only.yml new file mode 100644 index 000000000..518f18bde --- /dev/null +++ b/tests/unit/settings/user_settings_keep_only.yml @@ -0,0 +1,14 @@ +use_default_settings: + engines: + keep_only: + - wikibooks + - wikinews +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value +engines: + - name: wikipedia + - name: newengine + engine: dummy diff --git a/tests/unit/settings/user_settings_remove.yml b/tests/unit/settings/user_settings_remove.yml new file mode 100644 index 000000000..c4fd85df7 --- /dev/null +++ b/tests/unit/settings/user_settings_remove.yml @@ -0,0 +1,10 @@ +use_default_settings: + engines: + remove: + - wikibooks + - wikinews +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value diff --git a/tests/unit/settings/user_settings_remove2.yml b/tests/unit/settings/user_settings_remove2.yml new file mode 100644 index 000000000..e9be325dc --- /dev/null +++ b/tests/unit/settings/user_settings_remove2.yml @@ -0,0 +1,15 @@ +use_default_settings: + engines: + remove: + - wikibooks + - wikinews +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value +engines: + - name: wikipedia + tokens: ['secret_token'] + - name: newengine + engine: dummy diff --git a/tests/unit/settings/user_settings_simple.yml b/tests/unit/settings/user_settings_simple.yml new file mode 100644 index 000000000..36e5f1647 --- /dev/null +++ b/tests/unit/settings/user_settings_simple.yml @@ -0,0 +1,6 @@ +use_default_settings: True +server: + secret_key: "user_secret_key" + bind_address: "0.0.0.0" + default_http_headers: + Custom-Header: Custom-Value diff --git a/tests/unit/test_settings_loader.py b/tests/unit/test_settings_loader.py new file mode 100644 index 000000000..7df64e524 --- /dev/null +++ b/tests/unit/test_settings_loader.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +from os.path import dirname, join, abspath +from unittest.mock import patch + +from searx.testing import SearxTestCase +from searx.exceptions import SearxSettingsException +from searx import settings_loader + + +test_dir = abspath(dirname(__file__)) + + +class TestLoad(SearxTestCase): + + def test_load_zero(self): + with self.assertRaises(SearxSettingsException): + settings_loader.load_yaml('/dev/zero') + + with self.assertRaises(SearxSettingsException): + settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml')) + + with self.assertRaises(SearxSettingsException): + settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml')) + + def test_check_settings_yml(self): + self.assertIsNone(settings_loader.check_settings_yml('/dev/zero')) + + bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml') + self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path) + + +class TestDefaultSettings(SearxTestCase): + + def test_load(self): + settings, msg = settings_loader.load_settings(load_user_setttings=False) + self.assertTrue(msg.startswith('load the default settings from')) + self.assertFalse(settings['general']['debug']) + self.assertTrue(isinstance(settings['general']['instance_name'], str)) + self.assertEqual(settings['server']['secret_key'], "ultrasecretkey") + self.assertTrue(isinstance(settings['server']['port'], int)) + self.assertTrue(isinstance(settings['server']['bind_address'], str)) + self.assertTrue(isinstance(settings['engines'], list)) + self.assertTrue(isinstance(settings['locales'], dict)) + self.assertTrue(isinstance(settings['doi_resolvers'], dict)) + self.assertTrue(isinstance(settings['default_doi_resolver'], str)) + + +class TestUserSettings(SearxTestCase): + + def test_is_use_default_settings(self): + self.assertFalse(settings_loader.is_use_default_settings({})) + self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True})) + self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}})) + with self.assertRaises(ValueError): + self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1})) + with self.assertRaises(ValueError): + self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0})) + + def test_user_settings_not_found(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': '/dev/null'}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('load the default settings from')) + self.assertEqual(settings['server']['secret_key'], "ultrasecretkey") + + def test_user_settings(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + self.assertEqual(settings['server']['secret_key'], "user_secret_key") + self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value") + + def test_user_settings_remove(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + self.assertEqual(settings['server']['secret_key'], "user_secret_key") + self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value") + engine_names = [engine['name'] for engine in settings['engines']] + self.assertNotIn('wikinews', engine_names) + self.assertNotIn('wikibooks', engine_names) + self.assertIn('wikipedia', engine_names) + + def test_user_settings_remove2(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + self.assertEqual(settings['server']['secret_key'], "user_secret_key") + self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value") + engine_names = [engine['name'] for engine in settings['engines']] + self.assertNotIn('wikinews', engine_names) + self.assertNotIn('wikibooks', engine_names) + self.assertIn('wikipedia', engine_names) + wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines'])) + self.assertEqual(wikipedia[0]['engine'], 'wikipedia') + self.assertEqual(wikipedia[0]['tokens'], ['secret_token']) + newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines'])) + self.assertEqual(newengine[0]['engine'], 'dummy') + + def test_user_settings_keep_only(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('merge the default settings')) + engine_names = [engine['name'] for engine in settings['engines']] + self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine']) + # wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml + self.assertEqual(len(settings['engines'][2]), 1) + + def test_custom_settings(self): + with patch.dict(settings_loader.environ, + {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}): + settings, msg = settings_loader.load_settings() + self.assertTrue(msg.startswith('load the user settings from')) + self.assertEqual(settings['server']['port'], 9000) + self.assertEqual(settings['server']['secret_key'], "user_settings_secret") + engine_names = [engine['name'] for engine in settings['engines']] + self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote']) diff --git a/utils/update_user_settings.py b/utils/update_user_settings.py deleted file mode 100644 index fb6fd0b3f..000000000 --- a/utils/update_user_settings.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -# set path -from sys import path -from os.path import realpath, dirname, join -path.append(realpath(dirname(realpath(__file__)) + '/../')) - -import argparse -import sys -import string -import ruamel.yaml -import secrets -import collections -from ruamel.yaml.scalarstring import SingleQuotedScalarString, DoubleQuotedScalarString -from searx.settings import load_settings, check_settings_yml, get_default_settings_path -from searx.exceptions import SearxSettingsException - - -RANDOM_STRING_LETTERS = string.ascii_lowercase + string.digits + string.ascii_uppercase - - -def get_random_string(): - r = [secrets.choice(RANDOM_STRING_LETTERS) for _ in range(64)] - return ''.join(r) - - -def main(prog_arg): - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True - yaml.indent(mapping=4, sequence=1, offset=2) - user_settings_path = prog_args.get('user-settings-yaml') - - try: - default_settings, _ = load_settings(False) - if check_settings_yml(user_settings_path): - with open(user_settings_path, 'r', encoding='utf-8') as f: - user_settings = yaml.load(f.read()) - new_user_settings = False - else: - user_settings = yaml.load('use_default_settings: True') - new_user_settings = True - except SearxSettingsException as e: - sys.stderr.write(str(e)) - return - - if not new_user_settings and not user_settings.get('use_default_settings'): - sys.stderr.write('settings.yml already exists and use_default_settings is not True') - return - - user_settings['use_default_settings'] = True - use_default_settings_comment = "settings based on " + get_default_settings_path() - user_settings.yaml_add_eol_comment(use_default_settings_comment, 'use_default_settings') - - if user_settings.get('server', {}).get('secret_key') in [None, 'ultrasecretkey']: - user_settings.setdefault('server', {})['secret_key'] = DoubleQuotedScalarString(get_random_string()) - - user_engines = user_settings.get('engines') - if user_engines: - has_user_engines = True - user_engines_dict = dict((definition['name'], definition) for definition in user_engines) - else: - has_user_engines = False - user_engines_dict = {} - user_engines = [] - - # remove old engines - if prog_arg.get('add-engines') or has_user_engines: - default_engines_dict = dict((definition['name'], definition) for definition in default_settings['engines']) - for i, engine in enumerate(user_engines): - if engine['name'] not in default_engines_dict: - del user_engines[i] - - # add new engines - if prog_arg.get('add-engines'): - for engine in default_settings.get('engines', {}): - if engine['name'] not in user_engines_dict: - user_engines.append({'name': engine['name']}) - user_settings['engines'] = user_engines - - # output - if prog_arg.get('dry-run'): - yaml.dump(user_settings, sys.stdout) - else: - with open(user_settings_path, 'w', encoding='utf-8') as f: - yaml.dump(user_settings, f) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Update user settings.yml') - parser.add_argument('--add-engines', dest='add-engines', default=False, action='store_true', help='Add new engines') - parser.add_argument('--dry-run', dest='dry-run', default=False, action='store_true', help='Dry run') - parser.add_argument('user-settings-yaml', type=str) - return vars(parser.parse_args()) - - -if __name__ == '__main__': - prog_args = parse_args() - main(prog_args)