kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-22 12:10:11 +01:00
Code

[mod] move all default settings into searx.settings_defaults

Browse Source
This commit is contained in:
Alexandre Flament 2021-05-28 18:45:22 +02:00
parent 856729226d
commit 4b07df62e5
9 changed files with 249 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
searx/__init__.py
View File

@ -17,37 +17,18 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
import logging
import searx.settings_loader
from os import environ
from os.path import realpath, dirname, join, abspath, isfile
from searx.settings_defaults import settings_set_defaults
from os.path import dirname, abspath
searx_dir = abspath(dirname(__file__))
searx_parent_dir = abspath(dirname(dirname(__file__)))
engine_dir = dirname(realpath(__file__))
static_path = abspath(join(dirname(__file__), 'static'))
settings, settings_load_message = searx.settings_loader.load_settings()
if settings['ui']['static_path']:
static_path = settings['ui']['static_path']
'''
enable debug if
the environnement variable SEARX_DEBUG is 1 or true
(whatever the value in settings.yml)
or general.debug=True in settings.yml
disable debug if
the environnement variable SEARX_DEBUG is 0 or false
(whatever the value in settings.yml)
or general.debug=False in settings.yml
'''
searx_debug_env = environ.get('SEARX_DEBUG', '').lower()
if searx_debug_env == 'true' or searx_debug_env == '1':
searx_debug = True
elif searx_debug_env == 'false' or searx_debug_env == '0':
searx_debug = False
else:
searx_debug = settings.get('general', {}).get('debug')
if settings is not None:
settings = settings_set_defaults(settings)
searx_debug = settings['general']['debug']
if searx_debug:
logging.basicConfig(level=logging.DEBUG)
else:
@ -55,12 +36,13 @@ else:
logger = logging.getLogger('searx')
logger.info(settings_load_message)
logger.info('Initialisation done')
if 'SEARX_SECRET' in environ:
settings['server']['secret_key'] = environ['SEARX_SECRET']
if 'SEARX_BIND_ADDRESS' in environ:
settings['server']['bind_address'] = environ['SEARX_BIND_ADDRESS']
# log max_request_timeout
max_request_timeout = settings['outgoing']['max_request_timeout']
if max_request_timeout is None:
logger.info('max_request_timeout=%s', repr(max_request_timeout))
else:
logger.info('max_request_timeout=%i second(s)', max_request_timeout)
class _brand_namespace:

2
searx/engines/__init__.py
View File

@ -144,7 +144,7 @@ def load_engine(engine_data):
# exclude onion engines if not using tor.
return None
engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0)
engine.timeout += settings['outgoing']['extra_proxy_timeout']
for category_name in engine.categories:
categories.setdefault(category_name, []).append(engine)

28
searx/network/network.py
View File

@ -224,28 +224,22 @@ def initialize(settings_engines=None, settings_outgoing=None):
global NETWORKS
settings_engines = settings_engines or settings.get('engines')
settings_outgoing = settings_outgoing or settings.get('outgoing')
settings_engines = settings_engines or settings['engines']
settings_outgoing = settings_outgoing or settings['outgoing']
# default parameters for AsyncHTTPTransport
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
default_params = {
'enable_http': False,
'verify': True,
'enable_http2': settings_outgoing.get('enable_http2', True),
# Magic number kept from previous code
'max_connections': settings_outgoing.get('pool_connections', 100),
# Picked from constructor
'max_keepalive_connections': settings_outgoing.get('pool_maxsize', 10),
#
'keepalive_expiry': settings_outgoing.get('keepalive_expiry', 5.0),
'local_addresses': settings_outgoing.get('source_ips'),
'proxies': settings_outgoing.get('proxies'),
# default maximum redirect
# from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55
'max_redirects': settings_outgoing.get('max_redirects', 30),
#
'retries': settings_outgoing.get('retries', 0),
'enable_http2': settings_outgoing['enable_http2'],
'max_connections': settings_outgoing['pool_connections'],
'max_keepalive_connections': settings_outgoing['pool_maxsize'],
'keepalive_expiry': settings_outgoing['keepalive_expiry'],
'local_addresses': settings_outgoing['source_ips'],
'proxies': settings_outgoing['proxies'],
'max_redirects': settings_outgoing['max_redirects'],
'retries': settings_outgoing['retries'],
'retry_on_http_error': None,
}
@ -274,7 +268,7 @@ def initialize(settings_engines=None, settings_outgoing=None):
NETWORKS['ipv6'] = new_network({'local_addresses': '::'})
# define networks from outgoing.networks
for network_name, network in settings_outgoing.get('networks', {}).items():
for network_name, network in settings_outgoing['networks'].items():
NETWORKS[network_name] = new_network(network)
# define networks from engines.[i].network (except references)

10
searx/plugins/__init__.py
View File

@ -21,7 +21,7 @@ from os import listdir, makedirs, remove, stat, utime
from os.path import abspath, basename, dirname, exists, join
from shutil import copyfile
from searx import logger, settings, static_path
from searx import logger, settings
logger = logger.getChild('plugins')
@ -123,7 +123,7 @@ def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
def prepare_package_resources(pkg, name):
plugin_dir = 'plugin_' + name
target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)
target_dir = join(settings['ui']['static_path'], 'plugins/external_plugins', plugin_dir)
try:
makedirs(target_dir, exist_ok=True)
except:
@ -170,10 +170,10 @@ plugins.register(search_on_category_select)
plugins.register(tracker_url_remover)
plugins.register(vim_hotkeys)
# load external plugins
if 'plugins' in settings:
if settings['plugins']:
plugins.register(*settings['plugins'], external=True)
if 'enabled_plugins' in settings:
if settings['enabled_plugins']:
for plugin in plugins:
if plugin.name in settings['enabled_plugins']:
plugin.default_on = True
@ -181,5 +181,5 @@ if 'enabled_plugins' in settings:
plugin.default_on = False
# load tor specific plugins
if settings['outgoing'].get('using_tor_proxy'):
if settings['outgoing']['using_tor_proxy']:
plugins.register(ahmia_filter)

22
searx/preferences.py
View File

@ -333,25 +333,25 @@ class Preferences:
choices=categories + ['none']
),
'language': SearchLanguageSetting(
settings['search'].get('default_lang', ''),
settings['search']['default_lang'],
is_locked('language'),
choices=list(LANGUAGE_CODES) + ['']
),
'locale': EnumStringSetting(
settings['ui'].get('default_locale', ''),
settings['ui']['default_locale'],
is_locked('locale'),
choices=list(settings['locales'].keys()) + ['']
),
'autocomplete': EnumStringSetting(
settings['search'].get('autocomplete', ''),
settings['search']['autocomplete'],
is_locked('autocomplete'),
choices=list(autocomplete.backends.keys()) + ['']
),
'image_proxy': MapSetting(
settings['server'].get('image_proxy', False),
settings['server']['image_proxy'],
is_locked('image_proxy'),
map={
'': settings['server'].get('image_proxy', 0),
'': settings['server']['image_proxy'],
'0': False,
'1': True,
'True': True,
@ -359,12 +359,12 @@ class Preferences:
}
),
'method': EnumStringSetting(
settings['server'].get('method', 'POST'),
settings['server']['method'],
is_locked('method'),
choices=('GET', 'POST')
),
'safesearch': MapSetting(
settings['search'].get('safe_search', 0),
settings['search']['safe_search'],
is_locked('safesearch'),
map={
'0': 0,
@ -373,12 +373,12 @@ class Preferences:
}
),
'theme': EnumStringSetting(
settings['ui'].get('default_theme', 'oscar'),
settings['ui']['default_theme'],
is_locked('theme'),
choices=themes
),
'results_on_new_tab': MapSetting(
settings['ui'].get('results_on_new_tab', False),
settings['ui']['results_on_new_tab'],
is_locked('results_on_new_tab'),
map={
'0': False,
@ -393,11 +393,11 @@ class Preferences:
choices=DOI_RESOLVERS
),
'oscar-style': EnumStringSetting(
settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'),
settings['ui']['theme_args']['oscar_style'],
is_locked('oscar-style'),
choices=['', 'logicodev', 'logicodev-dark', 'pointhi']),
'advanced_search': MapSetting(
settings['ui'].get('advanced_search', False),
settings['ui']['advanced_search'],
is_locked('advanced_search'),
map={
'0': False,

194
searx/settings_defaults.py Normal file
View File

@ -0,0 +1,194 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pylint: disable=missing-function-docstring, missing-module-docstring
import typing
import numbers
import errno
import os
import logging
from os.path import dirname, abspath
from searx.languages import language_codes as languages
searx_dir = abspath(dirname(__file__))
logger = logging.getLogger('searx')
OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
LANGUAGE_CODES = ('', 'all') + tuple(l[0] for l in languages)
OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi')
CATEGORY_ORDER = [
'general',
'images',
'videos',
'news',
'map',
'music',
'it',
'science',
'files',
'social medias',
]
STR_TO_BOOL = {
'0': False,
'false': False,
'off': False,
'1': True,
'true': True,
'on': True,
}
_UNDEFINED = object()
class SettingsValue:
"""Check and update a setting value
"""
def __init__(self,
type_definition: typing.Union[None, typing.Any, typing.Tuple[typing.Any]]=None,
default: typing.Any=None,
environ_name: str=None):
self.type_definition = type_definition \
if type_definition is None or isinstance(type_definition, tuple) \
else (type_definition,)
self.default = default
self.environ_name = environ_name
@property
def type_definition_repr(self):
types_str = [t.__name__ if isinstance(t, type) else repr(t)
for t in self.type_definition]
return ', '.join(types_str)
def check_type_definition(self, value: typing.Any) -> None:
if value in self.type_definition:
return
type_list = tuple(t for t in self.type_definition if isinstance(t, type))
if not isinstance(value, type_list):
raise ValueError('The value has to be one of these types/values: {}'\
.format(self.type_definition_repr))
def __call__(self, value: typing.Any) -> typing.Any:
if value == _UNDEFINED:
value = self.default
# override existing value with environ
if self.environ_name and self.environ_name in os.environ:
value = os.environ[self.environ_name]
if self.type_definition == (bool,):
value = STR_TO_BOOL[value.lower()]
#
self.check_type_definition(value)
return value
class SettingsDirectoryValue(SettingsValue):
"""Check and update a setting value that is a directory path
"""
def check_type_definition(self, value: typing.Any) -> typing.Any:
super().check_type_definition(value)
if not os.path.isdir(value):
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), value)
def __call__(self, value: typing.Any) -> typing.Any:
if value == '':
value = self.default
return super().__call__(value)
def apply_schema(settings, schema, path_list):
error = False
for key, value in schema.items():
if isinstance(value, SettingsValue):
try:
settings[key] = value(settings.get(key, _UNDEFINED))
except Exception as e: # pylint: disable=broad-except
# don't stop now: check other values
logger.error('%s: %s', '.'.join([*path_list, key]), e)
error = True
elif isinstance(value, dict):
error = error or apply_schema(settings.setdefault(key, {}), schema[key], [*path_list, key])
else:
settings.setdefault(key, value)
if len(path_list) == 0 and error:
raise ValueError('Invalid settings.yml')
return error
SCHEMA = {
'general': {
'debug': SettingsValue(bool, False, 'SEARX_DEBUG'),
'instance_name': SettingsValue(str, 'searxng'),
'contact_url': SettingsValue((None, False, str), None),
},
'brand': {
},
'search': {
'safe_search': SettingsValue((0,1,2), 0),
'autocomplete': SettingsValue(str, ''),
'default_lang': SettingsValue(LANGUAGE_CODES, ''),
'ban_time_on_fail': SettingsValue(numbers.Real, 5),
'max_ban_time_on_fail': SettingsValue(numbers.Real, 120),
'formats': SettingsValue(list, OUTPUT_FORMATS),
},
'server': {
'port': SettingsValue(int, 8888),
'bind_address': SettingsValue(str, '127.0.0.1', 'SEARX_BIND_ADDRESS'),
'secret_key': SettingsValue(str, environ_name='SEARX_SECRET'),
'base_url': SettingsValue((False, str), False),
'image_proxy': SettingsValue(bool, False),
'http_protocol_version': SettingsValue(('1.0', '1.1'), '1.0'),
'method': SettingsValue(('POST', 'GET'), 'POST'),
'default_http_headers': SettingsValue(dict, {}),
},
'ui': {
'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')),
'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')),
'default_theme': SettingsValue(str, 'oscar'),
'default_locale': SettingsValue(str, ''),
'theme_args': {
'oscar_style': SettingsValue(OSCAR_STYLE, 'logicodev'),
},
'results_on_new_tab': SettingsValue(bool, False),
'advanced_search': SettingsValue(bool, False),
'categories_order': SettingsValue(list, CATEGORY_ORDER),
},
'preferences': {
'lock': SettingsValue(list, []),
},
'outgoing': {
'useragent_suffix': SettingsValue(str, ''),
'request_timeout': SettingsValue(numbers.Real, 3.0),
'enable_http2': SettingsValue(bool, True),
'max_request_timeout': SettingsValue((None, numbers.Real), None),
# Magic number kept from previous code
'pool_connections': SettingsValue(int, 100),
# Picked from constructor
'pool_maxsize': SettingsValue(int, 10),
'keepalive_expiry': SettingsValue(numbers.Real, 5.0),
# default maximum redirect
# from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55
'max_redirects': SettingsValue(int, 30),
'retries': SettingsValue(int, 0),
'proxies': SettingsValue((None, str, dict), None),
'source_ips': SettingsValue((None, str, list), None),
# Tor configuration
'using_tor_proxy': SettingsValue(bool, False),
'extra_proxy_timeout': SettingsValue(int, 0),
'networks': {
},
},
'plugins': SettingsValue((None, list), None),
'enabled_plugins': SettingsValue(list, []),
'checker': {
'off_when_debug': SettingsValue(bool, True),
},
'engines': SettingsValue(list, []),
'locales': SettingsValue(dict, {'en': 'English'}),
'doi_resolvers': {
},
}
def settings_set_defaults(settings):
apply_schema(settings, SCHEMA, [])
return settings

55
searx/utils.py
View File

@ -8,7 +8,6 @@ from os.path import splitext, join
from random import choice
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
from collections.abc import Mapping
from lxml import html
from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult
@ -46,7 +45,7 @@ def searx_useragent():
"""Return the searx User Agent"""
return 'searx/{searx_version} {suffix}'.format(
searx_version=VERSION_STRING,
suffix=settings['outgoing'].get('useragent_suffix', '')).strip()
suffix=settings['outgoing']['useragent_suffix'].strip())
def gen_useragent(os=None):
@ -501,58 +500,6 @@ def get_engine_from_settings(name):
return {}
NOT_EXISTS = object()
"""Singleton used by :py:obj:`get_value` if a key does not exists."""
def get_value(dictionary, *keys, default=NOT_EXISTS):
"""Return the value from a *deep* mapping type (e.g. the ``settings`` object
from yaml). If the path to the *key* does not exists a :py:obj:`NOT_EXISTS`
is returned (non ``KeyError`` exception is raised).
.. code: python
>>> from searx import settings
>>> from searx.utils import get_value, NOT_EXISTS
>>> get_value(settings, 'checker', 'additional_tests', 'rosebud', 'result_container')
['not_empty', ['one_title_contains', 'citizen kane']]
>>> get_value(settings, 'search', 'xxx') is NOT_EXISTS
True
>>> get_value(settings, 'search', 'formats')
['html', 'csv', 'json', 'rss']
The list returned from the ``search.format`` key is not a mapping type, you
can't traverse along non-mapping types. If you try it, you will get a
:py:ref:`NOT_EXISTS`:
.. code: python
>>> get_value(settings, 'search', 'format', 'csv') is NOT_EXISTS
True
>>> get_value(settings, 'search', 'formats')[1]
'csv'
For convenience you can replace :py:ref:`NOT_EXISTS` by a default value of
your choice:
.. code: python
if 'csv' in get_value(settings, 'search', 'formats', default=[]):
print("csv format is denied")
"""
obj = dictionary
for k in keys:
if not isinstance(obj, Mapping):
raise TypeError("expected mapping type, got %s" % type(obj))
obj = obj.get(k, default)
if obj is default:
return obj
return obj
def get_xpath(xpath_spec):
"""Return cached compiled XPath

46
searx/webapp.py
View File

@ -56,12 +56,12 @@ from flask_babel import (
)
from searx import logger
from searx import brand, static_path
from searx import brand
from searx import (
settings,
searx_dir,
searx_debug,
)
from searx.settings_defaults import OUTPUT_FORMATS
from searx.exceptions import SearxParameterException
from searx.engines import (
categories,
@ -71,7 +71,6 @@ from searx.engines import (
from searx.webutils import (
UnicodeWriter,
highlight_content,
get_resources_directory,
get_static_files,
get_result_templates,
get_themes,
@ -88,7 +87,6 @@ from searx.utils import (
gen_useragent,
dict_subset,
match_language,
get_value,
)
from searx.version import VERSION_STRING
from searx.query import RawTextQuery
@ -139,7 +137,7 @@ if sys.version_info[0] < 3:
logger = logger.getChild('webapp')
# serve pages with HTTP/1.1
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])
# check secret_key
if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
@ -147,25 +145,22 @@ if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
sys.exit(1)
# about static
static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
logger.debug('static directory is %s', static_path)
static_files = get_static_files(static_path)
logger.debug('static directory is %s', settings['ui']['static_path'])
static_files = get_static_files(settings['ui']['static_path'])
# about templates
logger.debug('templates directory is %s', settings['ui']['templates_path'])
default_theme = settings['ui']['default_theme']
templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
logger.debug('templates directory is %s', templates_path)
templates_path = settings['ui']['templates_path']
themes = get_themes(templates_path)
result_templates = get_result_templates(templates_path)
global_favicons = []
for indice, theme in enumerate(themes):
global_favicons.append([])
theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
theme_img_path = os.path.join(settings['ui']['static_path'], 'themes', theme, 'img', 'icons')
for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
global_favicons[indice].extend(filenames)
OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
STATS_SORT_PARAMETERS = {
'name': (False, 'name', ''),
'score': (True, 'score', 0),
@ -177,7 +172,7 @@ STATS_SORT_PARAMETERS = {
# Flask app
app = Flask(
__name__,
static_folder=static_path,
static_folder=settings['ui']['static_path'],
template_folder=templates_path
)
@ -517,8 +512,7 @@ def render(template_name, override_theme=None, **kwargs):
kwargs['preferences'] = request.preferences
kwargs['search_formats'] = [
x for x in get_value(
settings, 'search', 'formats', default=OUTPUT_FORMATS)
x for x in settings['search']['formats']
if x != 'html']
kwargs['brand'] = brand
@ -545,12 +539,7 @@ def render(template_name, override_theme=None, **kwargs):
def _get_ordered_categories():
ordered_categories = []
if 'categories_order' not in settings['ui']:
ordered_categories = ['general']
ordered_categories.extend(x for x in sorted(categories.keys()) if x != 'general')
return ordered_categories
ordered_categories = settings['ui']['categories_order']
ordered_categories = list(settings['ui']['categories_order'])
ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
return ordered_categories
@ -610,7 +599,7 @@ def pre_request():
@app.after_request
def add_default_headers(response):
# set default http headers
for header, value in settings['server'].get('default_http_headers', {}).items():
for header, value in settings['server']['default_http_headers'].items():
if header in response.headers:
continue
response.headers[header] = value
@ -696,7 +685,7 @@ def search():
if output_format not in OUTPUT_FORMATS:
output_format = 'html'
if output_format not in get_value(settings, 'search', 'formats', default=OUTPUT_FORMATS):
if output_format not in settings['search']['formats']:
flask.abort(403)
# check if there is query (not None and not an empty string)
@ -1069,11 +1058,6 @@ def preferences():
'time_range_support': time_range_support,
}
#
locked_preferences = list()
if 'preferences' in settings and 'lock' in settings['preferences']:
locked_preferences = settings['preferences']['lock']
#
return render('preferences.html',
selected_categories=get_selected_categories(request.preferences, request.form),
@ -1098,7 +1082,7 @@ def preferences():
theme=get_current_theme_name(),
preferences_url_params=request.preferences.get_as_url_params(),
base_url=get_base_url(),
locked_preferences=locked_preferences,
locked_preferences=settings['preferences']['lock'],
preferences=True)
@ -1271,7 +1255,7 @@ def favicon():
return send_from_directory(
os.path.join(
app.root_path,
static_path,
settings['ui']['static_path'],
'themes',
get_current_theme_name(),
'img'),

8
searx/webutils.py
View File

@ -47,14 +47,6 @@ class UnicodeWriter:
self.writerow(row)
def get_resources_directory(searx_directory, subdirectory, resources_directory):
if not resources_directory:
resources_directory = os.path.join(searx_directory, subdirectory)
if not os.path.isdir(resources_directory):
raise Exception(resources_directory + " is not a directory")
return resources_directory
def get_themes(templates_path):
"""Returns available themes list."""
themes = os.listdir(templates_path)