mirror of
https://github.com/searxng/searxng.git
synced 2024-08-22 22:20:42 +02:00
[enh] settings.yml: add use_default_settings option (2nd version)
This commit is contained in:
parent
1cfe7f2a75
commit
b4b81a5e1a
15
Makefile
15
Makefile
@ -266,19 +266,4 @@ test.clean:
|
|||||||
travis.codecov:
|
travis.codecov:
|
||||||
$(Q)$(PY_ENV_BIN)/python -m pip install codecov
|
$(Q)$(PY_ENV_BIN)/python -m pip install codecov
|
||||||
|
|
||||||
|
|
||||||
# user-settings
|
|
||||||
# -------------
|
|
||||||
|
|
||||||
PHONY += user-settings.create user-settings.update
|
|
||||||
|
|
||||||
user-settings.update: pyenvinstall
|
|
||||||
$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
|
|
||||||
$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH}
|
|
||||||
|
|
||||||
user-settings.update.engines: pyenvinstall
|
|
||||||
$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
|
|
||||||
$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH}
|
|
||||||
|
|
||||||
|
|
||||||
.PHONY: $(PHONY)
|
.PHONY: $(PHONY)
|
@ -235,68 +235,51 @@ In the following example, the actual settings are the default settings defined i
|
|||||||
|
|
||||||
.. code-block:: yaml
|
.. code-block:: yaml
|
||||||
|
|
||||||
use_default_settings: true
|
use_default_settings: True
|
||||||
server:
|
server:
|
||||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||||
server:
|
|
||||||
bind_address: "0.0.0.0"
|
bind_address: "0.0.0.0"
|
||||||
|
|
||||||
With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section:
|
With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
|
||||||
|
|
||||||
* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example).
|
In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
|
||||||
* If the ``engines`` section is defined then:
|
|
||||||
|
|
||||||
* searx loads only the engines declare in the user setttings.
|
|
||||||
* searx merges the configuration according to the engine name.
|
|
||||||
|
|
||||||
In the following example, only three engines are available. Each engine configuration is merged with the default configuration.
|
|
||||||
|
|
||||||
.. code-block:: yaml
|
.. code-block:: yaml
|
||||||
|
|
||||||
use_default_settings: true
|
use_default_settings: True
|
||||||
server:
|
|
||||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
|
||||||
engines:
|
|
||||||
- name: wikipedia
|
|
||||||
- name: wikidata
|
|
||||||
- name: ddg definitions
|
|
||||||
|
|
||||||
Another example where four engines are available. The arch linux wiki engine has a :ref:`token<private engines>`.
|
|
||||||
|
|
||||||
.. code-block:: yaml
|
|
||||||
|
|
||||||
use_default_settings: true
|
|
||||||
server:
|
server:
|
||||||
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||||
engines:
|
engines:
|
||||||
- name: arch linux wiki
|
- name: arch linux wiki
|
||||||
tokens: ['$ecretValue']
|
tokens: ['$ecretValue']
|
||||||
- name: wikipedia
|
|
||||||
- name: wikidata
|
|
||||||
- name: ddg definitions
|
|
||||||
|
|
||||||
automatic update
|
It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
|
||||||
----------------
|
|
||||||
|
|
||||||
The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case):
|
.. code-block:: yaml
|
||||||
|
|
||||||
.. code-block:: sh
|
use_default_settings:
|
||||||
|
engines:
|
||||||
|
remove:
|
||||||
|
- google
|
||||||
|
server:
|
||||||
|
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||||
|
engines:
|
||||||
|
- name: arch linux wiki
|
||||||
|
tokens: ['$ecretValue']
|
||||||
|
|
||||||
make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update
|
As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
|
||||||
|
|
||||||
Set ``SEARX_SETTINGS_PATH`` to your user settings path.
|
.. code-block:: yaml
|
||||||
|
|
||||||
As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated.
|
use_default_settings:
|
||||||
The following command creates or updates the user settings including the ``engines`` section:
|
engines:
|
||||||
|
keep_only:
|
||||||
.. code-block:: sh
|
- google
|
||||||
|
- duckduckgo
|
||||||
make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines
|
server:
|
||||||
|
secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
|
||||||
After that ``/etc/searx/settings.yml``
|
engines:
|
||||||
|
- name: google
|
||||||
* has a ``secret key``
|
tokens: ['$ecretValue']
|
||||||
* has a ``engine`` section if it is not already the case, moreover the command:
|
- name: duckduckgo
|
||||||
|
tokens: ['$ecretValue']
|
||||||
* has deleted engines that do not exist in the default settings.
|
|
||||||
* has added engines that exist in the default settings but are not declare in the user settings.
|
|
||||||
|
@ -16,7 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import searx.settings
|
import searx.settings_loader
|
||||||
from os import environ
|
from os import environ
|
||||||
from os.path import realpath, dirname, join, abspath, isfile
|
from os.path import realpath, dirname, join, abspath, isfile
|
||||||
|
|
||||||
@ -24,7 +24,7 @@ from os.path import realpath, dirname, join, abspath, isfile
|
|||||||
searx_dir = abspath(dirname(__file__))
|
searx_dir = abspath(dirname(__file__))
|
||||||
engine_dir = dirname(realpath(__file__))
|
engine_dir = dirname(realpath(__file__))
|
||||||
static_path = abspath(join(dirname(__file__), 'static'))
|
static_path = abspath(join(dirname(__file__), 'static'))
|
||||||
settings, settings_load_message = searx.settings.load_settings()
|
settings, settings_load_message = searx.settings_loader.load_settings()
|
||||||
|
|
||||||
if settings['ui']['static_path']:
|
if settings['ui']['static_path']:
|
||||||
static_path = settings['ui']['static_path']
|
static_path = settings['ui']['static_path']
|
||||||
|
@ -1,91 +0,0 @@
|
|||||||
import collections.abc
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
from searx.exceptions import SearxSettingsException
|
|
||||||
from os import environ
|
|
||||||
from os.path import dirname, join, abspath, isfile
|
|
||||||
|
|
||||||
|
|
||||||
searx_dir = abspath(dirname(__file__))
|
|
||||||
|
|
||||||
|
|
||||||
def check_settings_yml(file_name):
|
|
||||||
if isfile(file_name):
|
|
||||||
return file_name
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def load_yaml(file_name):
|
|
||||||
try:
|
|
||||||
with open(file_name, 'r', encoding='utf-8') as settings_yaml:
|
|
||||||
settings = yaml.safe_load(settings_yaml)
|
|
||||||
if not isinstance(settings, dict) or len(settings) == 0:
|
|
||||||
raise SearxSettingsException('Empty file', file_name)
|
|
||||||
return settings
|
|
||||||
except IOError as e:
|
|
||||||
raise SearxSettingsException(e, file_name)
|
|
||||||
except yaml.YAMLError as e:
|
|
||||||
raise SearxSettingsException(e, file_name)
|
|
||||||
|
|
||||||
|
|
||||||
def get_default_settings_path():
|
|
||||||
return check_settings_yml(join(searx_dir, 'settings.yml'))
|
|
||||||
|
|
||||||
|
|
||||||
def get_user_settings_path():
|
|
||||||
# find location of settings.yml
|
|
||||||
if 'SEARX_SETTINGS_PATH' in environ:
|
|
||||||
# if possible set path to settings using the
|
|
||||||
# enviroment variable SEARX_SETTINGS_PATH
|
|
||||||
return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
|
|
||||||
else:
|
|
||||||
# if not, get it from searx code base or last solution from /etc/searx
|
|
||||||
return check_settings_yml('/etc/searx/settings.yml')
|
|
||||||
|
|
||||||
|
|
||||||
def update_dict(d, u):
|
|
||||||
for k, v in u.items():
|
|
||||||
if isinstance(v, collections.abc.Mapping):
|
|
||||||
d[k] = update_dict(d.get(k, {}), v)
|
|
||||||
else:
|
|
||||||
d[k] = v
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
def update_settings(default_settings, user_settings):
|
|
||||||
for k, v in user_settings.items():
|
|
||||||
if k == 'use_default_settings':
|
|
||||||
continue
|
|
||||||
elif k == 'engines':
|
|
||||||
default_engines = default_settings[k]
|
|
||||||
default_engines_dict = dict((definition['name'], definition) for definition in default_engines)
|
|
||||||
default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition)
|
|
||||||
for definition in v]
|
|
||||||
else:
|
|
||||||
update_dict(default_settings[k], v)
|
|
||||||
|
|
||||||
return default_settings
|
|
||||||
|
|
||||||
|
|
||||||
def load_settings(load_user_setttings=True):
|
|
||||||
default_settings_path = get_default_settings_path()
|
|
||||||
user_settings_path = get_user_settings_path()
|
|
||||||
if user_settings_path is None or not load_user_setttings:
|
|
||||||
# no user settings
|
|
||||||
return (load_yaml(default_settings_path),
|
|
||||||
'load the default settings from {}'.format(default_settings_path))
|
|
||||||
|
|
||||||
# user settings
|
|
||||||
user_settings = load_yaml(user_settings_path)
|
|
||||||
if user_settings.get('use_default_settings'):
|
|
||||||
# the user settings are merged with the default configuration
|
|
||||||
default_settings = load_yaml(default_settings_path)
|
|
||||||
update_settings(default_settings, user_settings)
|
|
||||||
return (default_settings,
|
|
||||||
'merge the default settings ( {} ) and the user setttings ( {} )'
|
|
||||||
.format(default_settings_path, user_settings_path))
|
|
||||||
|
|
||||||
# the user settings, fully replace the default configuration
|
|
||||||
return (user_settings,
|
|
||||||
'load the user settings from {}'.format(user_settings_path))
|
|
129
searx/settings_loader.py
Normal file
129
searx/settings_loader.py
Normal file
@ -0,0 +1,129 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
from os import environ
|
||||||
|
from os.path import dirname, join, abspath, isfile
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from itertools import filterfalse
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from searx.exceptions import SearxSettingsException
|
||||||
|
|
||||||
|
|
||||||
|
searx_dir = abspath(dirname(__file__))
|
||||||
|
|
||||||
|
|
||||||
|
def check_settings_yml(file_name):
|
||||||
|
if isfile(file_name):
|
||||||
|
return file_name
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def load_yaml(file_name):
|
||||||
|
try:
|
||||||
|
with open(file_name, 'r', encoding='utf-8') as settings_yaml:
|
||||||
|
return yaml.safe_load(settings_yaml)
|
||||||
|
except IOError as e:
|
||||||
|
raise SearxSettingsException(e, file_name)
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
raise SearxSettingsException(e, file_name)
|
||||||
|
|
||||||
|
|
||||||
|
def get_default_settings_path():
|
||||||
|
return check_settings_yml(join(searx_dir, 'settings.yml'))
|
||||||
|
|
||||||
|
|
||||||
|
def get_user_settings_path():
|
||||||
|
# find location of settings.yml
|
||||||
|
if 'SEARX_SETTINGS_PATH' in environ:
|
||||||
|
# if possible set path to settings using the
|
||||||
|
# enviroment variable SEARX_SETTINGS_PATH
|
||||||
|
return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
|
||||||
|
|
||||||
|
# if not, get it from searx code base or last solution from /etc/searx
|
||||||
|
return check_settings_yml('/etc/searx/settings.yml')
|
||||||
|
|
||||||
|
|
||||||
|
def update_dict(default_dict, user_dict):
|
||||||
|
for k, v in user_dict.items():
|
||||||
|
if isinstance(v, Mapping):
|
||||||
|
default_dict[k] = update_dict(default_dict.get(k, {}), v)
|
||||||
|
else:
|
||||||
|
default_dict[k] = v
|
||||||
|
return default_dict
|
||||||
|
|
||||||
|
|
||||||
|
def update_settings(default_settings, user_settings):
|
||||||
|
# merge everything except the engines
|
||||||
|
for k, v in user_settings.items():
|
||||||
|
if k not in ('use_default_settings', 'engines'):
|
||||||
|
update_dict(default_settings[k], v)
|
||||||
|
|
||||||
|
# parse the engines
|
||||||
|
remove_engines = None
|
||||||
|
keep_only_engines = None
|
||||||
|
use_default_settings = user_settings.get('use_default_settings')
|
||||||
|
if isinstance(use_default_settings, dict):
|
||||||
|
remove_engines = use_default_settings.get('engines', {}).get('remove')
|
||||||
|
keep_only_engines = use_default_settings.get('engines', {}).get('keep_only')
|
||||||
|
|
||||||
|
if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None:
|
||||||
|
engines = default_settings['engines']
|
||||||
|
|
||||||
|
# parse "use_default_settings.engines.remove"
|
||||||
|
if remove_engines is not None:
|
||||||
|
engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines))
|
||||||
|
|
||||||
|
# parse "use_default_settings.engines.keep_only"
|
||||||
|
if keep_only_engines is not None:
|
||||||
|
engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines))
|
||||||
|
|
||||||
|
# parse "engines"
|
||||||
|
user_engines = user_settings.get('engines')
|
||||||
|
if user_engines:
|
||||||
|
engines_dict = dict((definition['name'], definition) for definition in engines)
|
||||||
|
for user_engine in user_engines:
|
||||||
|
default_engine = engines_dict.get(user_engine['name'])
|
||||||
|
if default_engine:
|
||||||
|
update_dict(default_engine, user_engine)
|
||||||
|
else:
|
||||||
|
engines.append(user_engine)
|
||||||
|
|
||||||
|
# store the result
|
||||||
|
default_settings['engines'] = engines
|
||||||
|
|
||||||
|
return default_settings
|
||||||
|
|
||||||
|
|
||||||
|
def is_use_default_settings(user_settings):
|
||||||
|
use_default_settings = user_settings.get('use_default_settings')
|
||||||
|
if use_default_settings is True:
|
||||||
|
return True
|
||||||
|
if isinstance(use_default_settings, dict):
|
||||||
|
return True
|
||||||
|
if use_default_settings is False or use_default_settings is None:
|
||||||
|
return False
|
||||||
|
raise ValueError('Invalid value for use_default_settings')
|
||||||
|
|
||||||
|
|
||||||
|
def load_settings(load_user_setttings=True):
|
||||||
|
default_settings_path = get_default_settings_path()
|
||||||
|
user_settings_path = get_user_settings_path()
|
||||||
|
if user_settings_path is None or not load_user_setttings:
|
||||||
|
# no user settings
|
||||||
|
return (load_yaml(default_settings_path),
|
||||||
|
'load the default settings from {}'.format(default_settings_path))
|
||||||
|
|
||||||
|
# user settings
|
||||||
|
user_settings = load_yaml(user_settings_path)
|
||||||
|
if is_use_default_settings(user_settings):
|
||||||
|
# the user settings are merged with the default configuration
|
||||||
|
default_settings = load_yaml(default_settings_path)
|
||||||
|
update_settings(default_settings, user_settings)
|
||||||
|
return (default_settings,
|
||||||
|
'merge the default settings ( {} ) and the user setttings ( {} )'
|
||||||
|
.format(default_settings_path, user_settings_path))
|
||||||
|
|
||||||
|
# the user settings, fully replace the default configuration
|
||||||
|
return (user_settings,
|
||||||
|
'load the user settings from {}'.format(user_settings_path))
|
0
tests/unit/settings/empty_settings.yml
Normal file
0
tests/unit/settings/empty_settings.yml
Normal file
2
tests/unit/settings/syntaxerror_settings.yml
Normal file
2
tests/unit/settings/syntaxerror_settings.yml
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
Test:
|
||||||
|
**********
|
111
tests/unit/settings/user_settings.yml
Normal file
111
tests/unit/settings/user_settings.yml
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
general:
|
||||||
|
debug : False
|
||||||
|
instance_name : "searx"
|
||||||
|
|
||||||
|
search:
|
||||||
|
safe_search : 0
|
||||||
|
autocomplete : ""
|
||||||
|
default_lang : ""
|
||||||
|
ban_time_on_fail : 5
|
||||||
|
max_ban_time_on_fail : 120
|
||||||
|
|
||||||
|
server:
|
||||||
|
port : 9000
|
||||||
|
bind_address : "0.0.0.0"
|
||||||
|
secret_key : "user_settings_secret"
|
||||||
|
base_url : False
|
||||||
|
image_proxy : False
|
||||||
|
http_protocol_version : "1.0"
|
||||||
|
method: "POST"
|
||||||
|
default_http_headers:
|
||||||
|
X-Content-Type-Options : nosniff
|
||||||
|
X-XSS-Protection : 1; mode=block
|
||||||
|
X-Download-Options : noopen
|
||||||
|
X-Robots-Tag : noindex, nofollow
|
||||||
|
Referrer-Policy : no-referrer
|
||||||
|
|
||||||
|
ui:
|
||||||
|
static_path : ""
|
||||||
|
templates_path : ""
|
||||||
|
default_theme : oscar
|
||||||
|
default_locale : ""
|
||||||
|
theme_args :
|
||||||
|
oscar_style : logicodev
|
||||||
|
|
||||||
|
engines:
|
||||||
|
- name : wikidata
|
||||||
|
engine : wikidata
|
||||||
|
shortcut : wd
|
||||||
|
timeout : 3.0
|
||||||
|
weight : 2
|
||||||
|
|
||||||
|
- name : wikibooks
|
||||||
|
engine : mediawiki
|
||||||
|
shortcut : wb
|
||||||
|
categories : general
|
||||||
|
base_url : "https://{language}.wikibooks.org/"
|
||||||
|
number_of_results : 5
|
||||||
|
search_type : text
|
||||||
|
|
||||||
|
- name : wikinews
|
||||||
|
engine : mediawiki
|
||||||
|
shortcut : wn
|
||||||
|
categories : news
|
||||||
|
base_url : "https://{language}.wikinews.org/"
|
||||||
|
number_of_results : 5
|
||||||
|
search_type : text
|
||||||
|
|
||||||
|
- name : wikiquote
|
||||||
|
engine : mediawiki
|
||||||
|
shortcut : wq
|
||||||
|
categories : general
|
||||||
|
base_url : "https://{language}.wikiquote.org/"
|
||||||
|
number_of_results : 5
|
||||||
|
search_type : text
|
||||||
|
|
||||||
|
locales:
|
||||||
|
en : English
|
||||||
|
ar : العَرَبِيَّة (Arabic)
|
||||||
|
bg : Български (Bulgarian)
|
||||||
|
bo : བོད་སྐད་ (Tibetian)
|
||||||
|
ca : Català (Catalan)
|
||||||
|
cs : Čeština (Czech)
|
||||||
|
cy : Cymraeg (Welsh)
|
||||||
|
da : Dansk (Danish)
|
||||||
|
de : Deutsch (German)
|
||||||
|
el_GR : Ελληνικά (Greek_Greece)
|
||||||
|
eo : Esperanto (Esperanto)
|
||||||
|
es : Español (Spanish)
|
||||||
|
et : Eesti (Estonian)
|
||||||
|
eu : Euskara (Basque)
|
||||||
|
fa_IR : (fārsī) فارسى (Persian)
|
||||||
|
fi : Suomi (Finnish)
|
||||||
|
fil : Wikang Filipino (Filipino)
|
||||||
|
fr : Français (French)
|
||||||
|
gl : Galego (Galician)
|
||||||
|
he : עברית (Hebrew)
|
||||||
|
hr : Hrvatski (Croatian)
|
||||||
|
hu : Magyar (Hungarian)
|
||||||
|
ia : Interlingua (Interlingua)
|
||||||
|
it : Italiano (Italian)
|
||||||
|
ja : 日本語 (Japanese)
|
||||||
|
lt : Lietuvių (Lithuanian)
|
||||||
|
nl : Nederlands (Dutch)
|
||||||
|
nl_BE : Vlaams (Dutch_Belgium)
|
||||||
|
oc : Lenga D'òc (Occitan)
|
||||||
|
pl : Polski (Polish)
|
||||||
|
pt : Português (Portuguese)
|
||||||
|
pt_BR : Português (Portuguese_Brazil)
|
||||||
|
ro : Română (Romanian)
|
||||||
|
ru : Русский (Russian)
|
||||||
|
sk : Slovenčina (Slovak)
|
||||||
|
sl : Slovenski (Slovene)
|
||||||
|
sr : српски (Serbian)
|
||||||
|
sv : Svenska (Swedish)
|
||||||
|
te : తెలుగు (telugu)
|
||||||
|
ta : தமிழ் (Tamil)
|
||||||
|
tr : Türkçe (Turkish)
|
||||||
|
uk : українська мова (Ukrainian)
|
||||||
|
vi : tiếng việt (Vietnamese)
|
||||||
|
zh : 中文 (Chinese)
|
||||||
|
zh_TW : 國語 (Taiwanese Mandarin)
|
14
tests/unit/settings/user_settings_keep_only.yml
Normal file
14
tests/unit/settings/user_settings_keep_only.yml
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
use_default_settings:
|
||||||
|
engines:
|
||||||
|
keep_only:
|
||||||
|
- wikibooks
|
||||||
|
- wikinews
|
||||||
|
server:
|
||||||
|
secret_key: "user_secret_key"
|
||||||
|
bind_address: "0.0.0.0"
|
||||||
|
default_http_headers:
|
||||||
|
Custom-Header: Custom-Value
|
||||||
|
engines:
|
||||||
|
- name: wikipedia
|
||||||
|
- name: newengine
|
||||||
|
engine: dummy
|
10
tests/unit/settings/user_settings_remove.yml
Normal file
10
tests/unit/settings/user_settings_remove.yml
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
use_default_settings:
|
||||||
|
engines:
|
||||||
|
remove:
|
||||||
|
- wikibooks
|
||||||
|
- wikinews
|
||||||
|
server:
|
||||||
|
secret_key: "user_secret_key"
|
||||||
|
bind_address: "0.0.0.0"
|
||||||
|
default_http_headers:
|
||||||
|
Custom-Header: Custom-Value
|
15
tests/unit/settings/user_settings_remove2.yml
Normal file
15
tests/unit/settings/user_settings_remove2.yml
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
use_default_settings:
|
||||||
|
engines:
|
||||||
|
remove:
|
||||||
|
- wikibooks
|
||||||
|
- wikinews
|
||||||
|
server:
|
||||||
|
secret_key: "user_secret_key"
|
||||||
|
bind_address: "0.0.0.0"
|
||||||
|
default_http_headers:
|
||||||
|
Custom-Header: Custom-Value
|
||||||
|
engines:
|
||||||
|
- name: wikipedia
|
||||||
|
tokens: ['secret_token']
|
||||||
|
- name: newengine
|
||||||
|
engine: dummy
|
6
tests/unit/settings/user_settings_simple.yml
Normal file
6
tests/unit/settings/user_settings_simple.yml
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
use_default_settings: True
|
||||||
|
server:
|
||||||
|
secret_key: "user_secret_key"
|
||||||
|
bind_address: "0.0.0.0"
|
||||||
|
default_http_headers:
|
||||||
|
Custom-Header: Custom-Value
|
122
tests/unit/test_settings_loader.py
Normal file
122
tests/unit/test_settings_loader.py
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
from os.path import dirname, join, abspath
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from searx.testing import SearxTestCase
|
||||||
|
from searx.exceptions import SearxSettingsException
|
||||||
|
from searx import settings_loader
|
||||||
|
|
||||||
|
|
||||||
|
test_dir = abspath(dirname(__file__))
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoad(SearxTestCase):
|
||||||
|
|
||||||
|
def test_load_zero(self):
|
||||||
|
with self.assertRaises(SearxSettingsException):
|
||||||
|
settings_loader.load_yaml('/dev/zero')
|
||||||
|
|
||||||
|
with self.assertRaises(SearxSettingsException):
|
||||||
|
settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml'))
|
||||||
|
|
||||||
|
with self.assertRaises(SearxSettingsException):
|
||||||
|
settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml'))
|
||||||
|
|
||||||
|
def test_check_settings_yml(self):
|
||||||
|
self.assertIsNone(settings_loader.check_settings_yml('/dev/zero'))
|
||||||
|
|
||||||
|
bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml')
|
||||||
|
self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDefaultSettings(SearxTestCase):
|
||||||
|
|
||||||
|
def test_load(self):
|
||||||
|
settings, msg = settings_loader.load_settings(load_user_setttings=False)
|
||||||
|
self.assertTrue(msg.startswith('load the default settings from'))
|
||||||
|
self.assertFalse(settings['general']['debug'])
|
||||||
|
self.assertTrue(isinstance(settings['general']['instance_name'], str))
|
||||||
|
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
|
||||||
|
self.assertTrue(isinstance(settings['server']['port'], int))
|
||||||
|
self.assertTrue(isinstance(settings['server']['bind_address'], str))
|
||||||
|
self.assertTrue(isinstance(settings['engines'], list))
|
||||||
|
self.assertTrue(isinstance(settings['locales'], dict))
|
||||||
|
self.assertTrue(isinstance(settings['doi_resolvers'], dict))
|
||||||
|
self.assertTrue(isinstance(settings['default_doi_resolver'], str))
|
||||||
|
|
||||||
|
|
||||||
|
class TestUserSettings(SearxTestCase):
|
||||||
|
|
||||||
|
def test_is_use_default_settings(self):
|
||||||
|
self.assertFalse(settings_loader.is_use_default_settings({}))
|
||||||
|
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True}))
|
||||||
|
self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}}))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1}))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0}))
|
||||||
|
|
||||||
|
def test_user_settings_not_found(self):
|
||||||
|
with patch.dict(settings_loader.environ,
|
||||||
|
{'SEARX_SETTINGS_PATH': '/dev/null'}):
|
||||||
|
settings, msg = settings_loader.load_settings()
|
||||||
|
self.assertTrue(msg.startswith('load the default settings from'))
|
||||||
|
self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
|
||||||
|
|
||||||
|
def test_user_settings(self):
|
||||||
|
with patch.dict(settings_loader.environ,
|
||||||
|
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}):
|
||||||
|
settings, msg = settings_loader.load_settings()
|
||||||
|
self.assertTrue(msg.startswith('merge the default settings'))
|
||||||
|
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
|
||||||
|
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
|
||||||
|
|
||||||
|
def test_user_settings_remove(self):
|
||||||
|
with patch.dict(settings_loader.environ,
|
||||||
|
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}):
|
||||||
|
settings, msg = settings_loader.load_settings()
|
||||||
|
self.assertTrue(msg.startswith('merge the default settings'))
|
||||||
|
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
|
||||||
|
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
|
||||||
|
engine_names = [engine['name'] for engine in settings['engines']]
|
||||||
|
self.assertNotIn('wikinews', engine_names)
|
||||||
|
self.assertNotIn('wikibooks', engine_names)
|
||||||
|
self.assertIn('wikipedia', engine_names)
|
||||||
|
|
||||||
|
def test_user_settings_remove2(self):
|
||||||
|
with patch.dict(settings_loader.environ,
|
||||||
|
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}):
|
||||||
|
settings, msg = settings_loader.load_settings()
|
||||||
|
self.assertTrue(msg.startswith('merge the default settings'))
|
||||||
|
self.assertEqual(settings['server']['secret_key'], "user_secret_key")
|
||||||
|
self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
|
||||||
|
engine_names = [engine['name'] for engine in settings['engines']]
|
||||||
|
self.assertNotIn('wikinews', engine_names)
|
||||||
|
self.assertNotIn('wikibooks', engine_names)
|
||||||
|
self.assertIn('wikipedia', engine_names)
|
||||||
|
wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines']))
|
||||||
|
self.assertEqual(wikipedia[0]['engine'], 'wikipedia')
|
||||||
|
self.assertEqual(wikipedia[0]['tokens'], ['secret_token'])
|
||||||
|
newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines']))
|
||||||
|
self.assertEqual(newengine[0]['engine'], 'dummy')
|
||||||
|
|
||||||
|
def test_user_settings_keep_only(self):
|
||||||
|
with patch.dict(settings_loader.environ,
|
||||||
|
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}):
|
||||||
|
settings, msg = settings_loader.load_settings()
|
||||||
|
self.assertTrue(msg.startswith('merge the default settings'))
|
||||||
|
engine_names = [engine['name'] for engine in settings['engines']]
|
||||||
|
self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine'])
|
||||||
|
# wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml
|
||||||
|
self.assertEqual(len(settings['engines'][2]), 1)
|
||||||
|
|
||||||
|
def test_custom_settings(self):
|
||||||
|
with patch.dict(settings_loader.environ,
|
||||||
|
{'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}):
|
||||||
|
settings, msg = settings_loader.load_settings()
|
||||||
|
self.assertTrue(msg.startswith('load the user settings from'))
|
||||||
|
self.assertEqual(settings['server']['port'], 9000)
|
||||||
|
self.assertEqual(settings['server']['secret_key'], "user_settings_secret")
|
||||||
|
engine_names = [engine['name'] for engine in settings['engines']]
|
||||||
|
self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote'])
|
@ -1,98 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# set path
|
|
||||||
from sys import path
|
|
||||||
from os.path import realpath, dirname, join
|
|
||||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
import string
|
|
||||||
import ruamel.yaml
|
|
||||||
import secrets
|
|
||||||
import collections
|
|
||||||
from ruamel.yaml.scalarstring import SingleQuotedScalarString, DoubleQuotedScalarString
|
|
||||||
from searx.settings import load_settings, check_settings_yml, get_default_settings_path
|
|
||||||
from searx.exceptions import SearxSettingsException
|
|
||||||
|
|
||||||
|
|
||||||
RANDOM_STRING_LETTERS = string.ascii_lowercase + string.digits + string.ascii_uppercase
|
|
||||||
|
|
||||||
|
|
||||||
def get_random_string():
|
|
||||||
r = [secrets.choice(RANDOM_STRING_LETTERS) for _ in range(64)]
|
|
||||||
return ''.join(r)
|
|
||||||
|
|
||||||
|
|
||||||
def main(prog_arg):
|
|
||||||
yaml = ruamel.yaml.YAML()
|
|
||||||
yaml.preserve_quotes = True
|
|
||||||
yaml.indent(mapping=4, sequence=1, offset=2)
|
|
||||||
user_settings_path = prog_args.get('user-settings-yaml')
|
|
||||||
|
|
||||||
try:
|
|
||||||
default_settings, _ = load_settings(False)
|
|
||||||
if check_settings_yml(user_settings_path):
|
|
||||||
with open(user_settings_path, 'r', encoding='utf-8') as f:
|
|
||||||
user_settings = yaml.load(f.read())
|
|
||||||
new_user_settings = False
|
|
||||||
else:
|
|
||||||
user_settings = yaml.load('use_default_settings: True')
|
|
||||||
new_user_settings = True
|
|
||||||
except SearxSettingsException as e:
|
|
||||||
sys.stderr.write(str(e))
|
|
||||||
return
|
|
||||||
|
|
||||||
if not new_user_settings and not user_settings.get('use_default_settings'):
|
|
||||||
sys.stderr.write('settings.yml already exists and use_default_settings is not True')
|
|
||||||
return
|
|
||||||
|
|
||||||
user_settings['use_default_settings'] = True
|
|
||||||
use_default_settings_comment = "settings based on " + get_default_settings_path()
|
|
||||||
user_settings.yaml_add_eol_comment(use_default_settings_comment, 'use_default_settings')
|
|
||||||
|
|
||||||
if user_settings.get('server', {}).get('secret_key') in [None, 'ultrasecretkey']:
|
|
||||||
user_settings.setdefault('server', {})['secret_key'] = DoubleQuotedScalarString(get_random_string())
|
|
||||||
|
|
||||||
user_engines = user_settings.get('engines')
|
|
||||||
if user_engines:
|
|
||||||
has_user_engines = True
|
|
||||||
user_engines_dict = dict((definition['name'], definition) for definition in user_engines)
|
|
||||||
else:
|
|
||||||
has_user_engines = False
|
|
||||||
user_engines_dict = {}
|
|
||||||
user_engines = []
|
|
||||||
|
|
||||||
# remove old engines
|
|
||||||
if prog_arg.get('add-engines') or has_user_engines:
|
|
||||||
default_engines_dict = dict((definition['name'], definition) for definition in default_settings['engines'])
|
|
||||||
for i, engine in enumerate(user_engines):
|
|
||||||
if engine['name'] not in default_engines_dict:
|
|
||||||
del user_engines[i]
|
|
||||||
|
|
||||||
# add new engines
|
|
||||||
if prog_arg.get('add-engines'):
|
|
||||||
for engine in default_settings.get('engines', {}):
|
|
||||||
if engine['name'] not in user_engines_dict:
|
|
||||||
user_engines.append({'name': engine['name']})
|
|
||||||
user_settings['engines'] = user_engines
|
|
||||||
|
|
||||||
# output
|
|
||||||
if prog_arg.get('dry-run'):
|
|
||||||
yaml.dump(user_settings, sys.stdout)
|
|
||||||
else:
|
|
||||||
with open(user_settings_path, 'w', encoding='utf-8') as f:
|
|
||||||
yaml.dump(user_settings, f)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
|
||||||
parser = argparse.ArgumentParser(description='Update user settings.yml')
|
|
||||||
parser.add_argument('--add-engines', dest='add-engines', default=False, action='store_true', help='Add new engines')
|
|
||||||
parser.add_argument('--dry-run', dest='dry-run', default=False, action='store_true', help='Dry run')
|
|
||||||
parser.add_argument('user-settings-yaml', type=str)
|
|
||||||
return vars(parser.parse_args())
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
prog_args = parse_args()
|
|
||||||
main(prog_args)
|
|
Loading…
Reference in New Issue
Block a user