mirror of
https://github.com/searxng/searxng.git
synced 2024-09-12 07:30:22 +02:00
Merge pull request #110 from searxng/mod-default-settings
[mod] move all default settings into searx.settings_defaults
This commit is contained in:
commit
e3f4a77311
@ -1,53 +1,21 @@
|
|||||||
'''
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
searx is free software: you can redistribute it and/or modify
|
# lint: pylint
|
||||||
it under the terms of the GNU Affero General Public License as published by
|
# pylint: disable=missing-function-docstring, missing-module-docstring
|
||||||
the Free Software Foundation, either version 3 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
searx is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU Affero General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU Affero General Public License
|
|
||||||
along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
|
||||||
|
|
||||||
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
from os.path import dirname, abspath
|
||||||
import logging
|
import logging
|
||||||
import searx.settings_loader
|
|
||||||
from os import environ
|
|
||||||
from os.path import realpath, dirname, join, abspath, isfile
|
|
||||||
|
|
||||||
|
import searx.settings_loader
|
||||||
|
from searx.settings_defaults import settings_set_defaults
|
||||||
|
|
||||||
searx_dir = abspath(dirname(__file__))
|
searx_dir = abspath(dirname(__file__))
|
||||||
searx_parent_dir = abspath(dirname(dirname(__file__)))
|
searx_parent_dir = abspath(dirname(dirname(__file__)))
|
||||||
engine_dir = dirname(realpath(__file__))
|
|
||||||
static_path = abspath(join(dirname(__file__), 'static'))
|
|
||||||
settings, settings_load_message = searx.settings_loader.load_settings()
|
settings, settings_load_message = searx.settings_loader.load_settings()
|
||||||
|
|
||||||
if settings['ui']['static_path']:
|
if settings is not None:
|
||||||
static_path = settings['ui']['static_path']
|
settings = settings_set_defaults(settings)
|
||||||
|
|
||||||
'''
|
|
||||||
enable debug if
|
|
||||||
the environnement variable SEARX_DEBUG is 1 or true
|
|
||||||
(whatever the value in settings.yml)
|
|
||||||
or general.debug=True in settings.yml
|
|
||||||
disable debug if
|
|
||||||
the environnement variable SEARX_DEBUG is 0 or false
|
|
||||||
(whatever the value in settings.yml)
|
|
||||||
or general.debug=False in settings.yml
|
|
||||||
'''
|
|
||||||
searx_debug_env = environ.get('SEARX_DEBUG', '').lower()
|
|
||||||
if searx_debug_env == 'true' or searx_debug_env == '1':
|
|
||||||
searx_debug = True
|
|
||||||
elif searx_debug_env == 'false' or searx_debug_env == '0':
|
|
||||||
searx_debug = False
|
|
||||||
else:
|
|
||||||
searx_debug = settings.get('general', {}).get('debug')
|
|
||||||
|
|
||||||
|
searx_debug = settings['general']['debug']
|
||||||
if searx_debug:
|
if searx_debug:
|
||||||
logging.basicConfig(level=logging.DEBUG)
|
logging.basicConfig(level=logging.DEBUG)
|
||||||
else:
|
else:
|
||||||
@ -55,15 +23,16 @@ else:
|
|||||||
|
|
||||||
logger = logging.getLogger('searx')
|
logger = logging.getLogger('searx')
|
||||||
logger.info(settings_load_message)
|
logger.info(settings_load_message)
|
||||||
logger.info('Initialisation done')
|
|
||||||
|
|
||||||
if 'SEARX_SECRET' in environ:
|
# log max_request_timeout
|
||||||
settings['server']['secret_key'] = environ['SEARX_SECRET']
|
max_request_timeout = settings['outgoing']['max_request_timeout']
|
||||||
if 'SEARX_BIND_ADDRESS' in environ:
|
if max_request_timeout is None:
|
||||||
settings['server']['bind_address'] = environ['SEARX_BIND_ADDRESS']
|
logger.info('max_request_timeout=%s', repr(max_request_timeout))
|
||||||
|
else:
|
||||||
|
logger.info('max_request_timeout=%i second(s)', max_request_timeout)
|
||||||
|
|
||||||
|
|
||||||
class _brand_namespace:
|
class _brand_namespace: # pylint: disable=invalid-name
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_val(cls, group, name, default=''):
|
def get_val(cls, group, name, default=''):
|
||||||
|
@ -144,7 +144,7 @@ def load_engine(engine_data):
|
|||||||
# exclude onion engines if not using tor.
|
# exclude onion engines if not using tor.
|
||||||
return None
|
return None
|
||||||
|
|
||||||
engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0)
|
engine.timeout += settings['outgoing']['extra_proxy_timeout']
|
||||||
|
|
||||||
for category_name in engine.categories:
|
for category_name in engine.categories:
|
||||||
categories.setdefault(category_name, []).append(engine)
|
categories.setdefault(category_name, []).append(engine)
|
||||||
|
@ -224,28 +224,22 @@ def initialize(settings_engines=None, settings_outgoing=None):
|
|||||||
|
|
||||||
global NETWORKS
|
global NETWORKS
|
||||||
|
|
||||||
settings_engines = settings_engines or settings.get('engines')
|
settings_engines = settings_engines or settings['engines']
|
||||||
settings_outgoing = settings_outgoing or settings.get('outgoing')
|
settings_outgoing = settings_outgoing or settings['outgoing']
|
||||||
|
|
||||||
# default parameters for AsyncHTTPTransport
|
# default parameters for AsyncHTTPTransport
|
||||||
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
|
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
|
||||||
default_params = {
|
default_params = {
|
||||||
'enable_http': False,
|
'enable_http': False,
|
||||||
'verify': True,
|
'verify': True,
|
||||||
'enable_http2': settings_outgoing.get('enable_http2', True),
|
'enable_http2': settings_outgoing['enable_http2'],
|
||||||
# Magic number kept from previous code
|
'max_connections': settings_outgoing['pool_connections'],
|
||||||
'max_connections': settings_outgoing.get('pool_connections', 100),
|
'max_keepalive_connections': settings_outgoing['pool_maxsize'],
|
||||||
# Picked from constructor
|
'keepalive_expiry': settings_outgoing['keepalive_expiry'],
|
||||||
'max_keepalive_connections': settings_outgoing.get('pool_maxsize', 10),
|
'local_addresses': settings_outgoing['source_ips'],
|
||||||
#
|
'proxies': settings_outgoing['proxies'],
|
||||||
'keepalive_expiry': settings_outgoing.get('keepalive_expiry', 5.0),
|
'max_redirects': settings_outgoing['max_redirects'],
|
||||||
'local_addresses': settings_outgoing.get('source_ips'),
|
'retries': settings_outgoing['retries'],
|
||||||
'proxies': settings_outgoing.get('proxies'),
|
|
||||||
# default maximum redirect
|
|
||||||
# from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55
|
|
||||||
'max_redirects': settings_outgoing.get('max_redirects', 30),
|
|
||||||
#
|
|
||||||
'retries': settings_outgoing.get('retries', 0),
|
|
||||||
'retry_on_http_error': None,
|
'retry_on_http_error': None,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -274,7 +268,7 @@ def initialize(settings_engines=None, settings_outgoing=None):
|
|||||||
NETWORKS['ipv6'] = new_network({'local_addresses': '::'})
|
NETWORKS['ipv6'] = new_network({'local_addresses': '::'})
|
||||||
|
|
||||||
# define networks from outgoing.networks
|
# define networks from outgoing.networks
|
||||||
for network_name, network in settings_outgoing.get('networks', {}).items():
|
for network_name, network in settings_outgoing['networks'].items():
|
||||||
NETWORKS[network_name] = new_network(network)
|
NETWORKS[network_name] = new_network(network)
|
||||||
|
|
||||||
# define networks from engines.[i].network (except references)
|
# define networks from engines.[i].network (except references)
|
||||||
|
@ -21,7 +21,7 @@ from os import listdir, makedirs, remove, stat, utime
|
|||||||
from os.path import abspath, basename, dirname, exists, join
|
from os.path import abspath, basename, dirname, exists, join
|
||||||
from shutil import copyfile
|
from shutil import copyfile
|
||||||
|
|
||||||
from searx import logger, settings, static_path
|
from searx import logger, settings
|
||||||
|
|
||||||
|
|
||||||
logger = logger.getChild('plugins')
|
logger = logger.getChild('plugins')
|
||||||
@ -123,7 +123,7 @@ def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):
|
|||||||
|
|
||||||
def prepare_package_resources(pkg, name):
|
def prepare_package_resources(pkg, name):
|
||||||
plugin_dir = 'plugin_' + name
|
plugin_dir = 'plugin_' + name
|
||||||
target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)
|
target_dir = join(settings['ui']['static_path'], 'plugins/external_plugins', plugin_dir)
|
||||||
try:
|
try:
|
||||||
makedirs(target_dir, exist_ok=True)
|
makedirs(target_dir, exist_ok=True)
|
||||||
except:
|
except:
|
||||||
@ -170,10 +170,10 @@ plugins.register(search_on_category_select)
|
|||||||
plugins.register(tracker_url_remover)
|
plugins.register(tracker_url_remover)
|
||||||
plugins.register(vim_hotkeys)
|
plugins.register(vim_hotkeys)
|
||||||
# load external plugins
|
# load external plugins
|
||||||
if 'plugins' in settings:
|
if settings['plugins']:
|
||||||
plugins.register(*settings['plugins'], external=True)
|
plugins.register(*settings['plugins'], external=True)
|
||||||
|
|
||||||
if 'enabled_plugins' in settings:
|
if settings['enabled_plugins']:
|
||||||
for plugin in plugins:
|
for plugin in plugins:
|
||||||
if plugin.name in settings['enabled_plugins']:
|
if plugin.name in settings['enabled_plugins']:
|
||||||
plugin.default_on = True
|
plugin.default_on = True
|
||||||
@ -181,5 +181,5 @@ if 'enabled_plugins' in settings:
|
|||||||
plugin.default_on = False
|
plugin.default_on = False
|
||||||
|
|
||||||
# load tor specific plugins
|
# load tor specific plugins
|
||||||
if settings['outgoing'].get('using_tor_proxy'):
|
if settings['outgoing']['using_tor_proxy']:
|
||||||
plugins.register(ahmia_filter)
|
plugins.register(ahmia_filter)
|
||||||
|
@ -333,25 +333,25 @@ class Preferences:
|
|||||||
choices=categories + ['none']
|
choices=categories + ['none']
|
||||||
),
|
),
|
||||||
'language': SearchLanguageSetting(
|
'language': SearchLanguageSetting(
|
||||||
settings['search'].get('default_lang', ''),
|
settings['search']['default_lang'],
|
||||||
is_locked('language'),
|
is_locked('language'),
|
||||||
choices=list(LANGUAGE_CODES) + ['']
|
choices=list(LANGUAGE_CODES) + ['']
|
||||||
),
|
),
|
||||||
'locale': EnumStringSetting(
|
'locale': EnumStringSetting(
|
||||||
settings['ui'].get('default_locale', ''),
|
settings['ui']['default_locale'],
|
||||||
is_locked('locale'),
|
is_locked('locale'),
|
||||||
choices=list(settings['locales'].keys()) + ['']
|
choices=list(settings['locales'].keys()) + ['']
|
||||||
),
|
),
|
||||||
'autocomplete': EnumStringSetting(
|
'autocomplete': EnumStringSetting(
|
||||||
settings['search'].get('autocomplete', ''),
|
settings['search']['autocomplete'],
|
||||||
is_locked('autocomplete'),
|
is_locked('autocomplete'),
|
||||||
choices=list(autocomplete.backends.keys()) + ['']
|
choices=list(autocomplete.backends.keys()) + ['']
|
||||||
),
|
),
|
||||||
'image_proxy': MapSetting(
|
'image_proxy': MapSetting(
|
||||||
settings['server'].get('image_proxy', False),
|
settings['server']['image_proxy'],
|
||||||
is_locked('image_proxy'),
|
is_locked('image_proxy'),
|
||||||
map={
|
map={
|
||||||
'': settings['server'].get('image_proxy', 0),
|
'': settings['server']['image_proxy'],
|
||||||
'0': False,
|
'0': False,
|
||||||
'1': True,
|
'1': True,
|
||||||
'True': True,
|
'True': True,
|
||||||
@ -359,12 +359,12 @@ class Preferences:
|
|||||||
}
|
}
|
||||||
),
|
),
|
||||||
'method': EnumStringSetting(
|
'method': EnumStringSetting(
|
||||||
settings['server'].get('method', 'POST'),
|
settings['server']['method'],
|
||||||
is_locked('method'),
|
is_locked('method'),
|
||||||
choices=('GET', 'POST')
|
choices=('GET', 'POST')
|
||||||
),
|
),
|
||||||
'safesearch': MapSetting(
|
'safesearch': MapSetting(
|
||||||
settings['search'].get('safe_search', 0),
|
settings['search']['safe_search'],
|
||||||
is_locked('safesearch'),
|
is_locked('safesearch'),
|
||||||
map={
|
map={
|
||||||
'0': 0,
|
'0': 0,
|
||||||
@ -373,12 +373,12 @@ class Preferences:
|
|||||||
}
|
}
|
||||||
),
|
),
|
||||||
'theme': EnumStringSetting(
|
'theme': EnumStringSetting(
|
||||||
settings['ui'].get('default_theme', 'oscar'),
|
settings['ui']['default_theme'],
|
||||||
is_locked('theme'),
|
is_locked('theme'),
|
||||||
choices=themes
|
choices=themes
|
||||||
),
|
),
|
||||||
'results_on_new_tab': MapSetting(
|
'results_on_new_tab': MapSetting(
|
||||||
settings['ui'].get('results_on_new_tab', False),
|
settings['ui']['results_on_new_tab'],
|
||||||
is_locked('results_on_new_tab'),
|
is_locked('results_on_new_tab'),
|
||||||
map={
|
map={
|
||||||
'0': False,
|
'0': False,
|
||||||
@ -393,11 +393,11 @@ class Preferences:
|
|||||||
choices=DOI_RESOLVERS
|
choices=DOI_RESOLVERS
|
||||||
),
|
),
|
||||||
'oscar-style': EnumStringSetting(
|
'oscar-style': EnumStringSetting(
|
||||||
settings['ui'].get('theme_args', {}).get('oscar_style', 'logicodev'),
|
settings['ui']['theme_args']['oscar_style'],
|
||||||
is_locked('oscar-style'),
|
is_locked('oscar-style'),
|
||||||
choices=['', 'logicodev', 'logicodev-dark', 'pointhi']),
|
choices=['', 'logicodev', 'logicodev-dark', 'pointhi']),
|
||||||
'advanced_search': MapSetting(
|
'advanced_search': MapSetting(
|
||||||
settings['ui'].get('advanced_search', False),
|
settings['ui']['advanced_search'],
|
||||||
is_locked('advanced_search'),
|
is_locked('advanced_search'),
|
||||||
map={
|
map={
|
||||||
'0': False,
|
'0': False,
|
||||||
|
@ -23,17 +23,6 @@ from searx.search.checker import initialize as initialize_checker
|
|||||||
|
|
||||||
logger = logger.getChild('search')
|
logger = logger.getChild('search')
|
||||||
|
|
||||||
max_request_timeout = settings.get('outgoing', {}).get('max_request_timeout' or None)
|
|
||||||
if max_request_timeout is None:
|
|
||||||
logger.info('max_request_timeout={0}'.format(max_request_timeout))
|
|
||||||
else:
|
|
||||||
if isinstance(max_request_timeout, float):
|
|
||||||
logger.info('max_request_timeout={0} second(s)'.format(max_request_timeout))
|
|
||||||
else:
|
|
||||||
logger.critical('outgoing.max_request_timeout if defined has to be float')
|
|
||||||
import sys
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
def initialize(settings_engines=None, enable_checker=False):
|
def initialize(settings_engines=None, enable_checker=False):
|
||||||
settings_engines = settings_engines or settings['engines']
|
settings_engines = settings_engines or settings['engines']
|
||||||
@ -115,6 +104,7 @@ class Search:
|
|||||||
default_timeout = max(default_timeout, processor.engine.timeout)
|
default_timeout = max(default_timeout, processor.engine.timeout)
|
||||||
|
|
||||||
# adjust timeout
|
# adjust timeout
|
||||||
|
max_request_timeout = settings['outgoing']['max_request_timeout']
|
||||||
actual_timeout = default_timeout
|
actual_timeout = default_timeout
|
||||||
query_timeout = self.search_query.timeout_limit
|
query_timeout = self.search_query.timeout_limit
|
||||||
|
|
||||||
|
202
searx/settings_defaults.py
Normal file
202
searx/settings_defaults.py
Normal file
@ -0,0 +1,202 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
# lint: pylint
|
||||||
|
# pylint: disable=missing-function-docstring
|
||||||
|
"""Implementation of the default settings.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import typing
|
||||||
|
import numbers
|
||||||
|
import errno
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
from os.path import dirname, abspath
|
||||||
|
|
||||||
|
from searx.languages import language_codes as languages
|
||||||
|
|
||||||
|
searx_dir = abspath(dirname(__file__))
|
||||||
|
|
||||||
|
logger = logging.getLogger('searx')
|
||||||
|
OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
|
||||||
|
LANGUAGE_CODES = ('', 'all') + tuple(l[0] for l in languages)
|
||||||
|
OSCAR_STYLE = ('logicodev', 'logicodev-dark', 'pointhi')
|
||||||
|
CATEGORY_ORDER = [
|
||||||
|
'general',
|
||||||
|
'images',
|
||||||
|
'videos',
|
||||||
|
'news',
|
||||||
|
'map',
|
||||||
|
'music',
|
||||||
|
'it',
|
||||||
|
'science',
|
||||||
|
'files',
|
||||||
|
'social medias',
|
||||||
|
]
|
||||||
|
STR_TO_BOOL = {
|
||||||
|
'0': False,
|
||||||
|
'false': False,
|
||||||
|
'off': False,
|
||||||
|
'1': True,
|
||||||
|
'true': True,
|
||||||
|
'on': True,
|
||||||
|
}
|
||||||
|
_UNDEFINED = object()
|
||||||
|
|
||||||
|
|
||||||
|
class SettingsValue:
|
||||||
|
"""Check and update a setting value
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
type_definition: typing.Union[None, typing.Any, typing.Tuple[typing.Any]]=None,
|
||||||
|
default: typing.Any=None,
|
||||||
|
environ_name: str=None):
|
||||||
|
self.type_definition = (
|
||||||
|
type_definition
|
||||||
|
if type_definition is None or isinstance(type_definition, tuple)
|
||||||
|
else (type_definition,)
|
||||||
|
)
|
||||||
|
self.default = default
|
||||||
|
self.environ_name = environ_name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def type_definition_repr(self):
|
||||||
|
types_str = [
|
||||||
|
t.__name__ if isinstance(t, type) else repr(t)
|
||||||
|
for t in self.type_definition
|
||||||
|
]
|
||||||
|
return ', '.join(types_str)
|
||||||
|
|
||||||
|
def check_type_definition(self, value: typing.Any) -> None:
|
||||||
|
if value in self.type_definition:
|
||||||
|
return
|
||||||
|
type_list = tuple(t for t in self.type_definition if isinstance(t, type))
|
||||||
|
if not isinstance(value, type_list):
|
||||||
|
raise ValueError(
|
||||||
|
'The value has to be one of these types/values: {}'.format(
|
||||||
|
self.type_definition_repr))
|
||||||
|
|
||||||
|
def __call__(self, value: typing.Any) -> typing.Any:
|
||||||
|
if value == _UNDEFINED:
|
||||||
|
value = self.default
|
||||||
|
# override existing value with environ
|
||||||
|
if self.environ_name and self.environ_name in os.environ:
|
||||||
|
value = os.environ[self.environ_name]
|
||||||
|
if self.type_definition == (bool,):
|
||||||
|
value = STR_TO_BOOL[value.lower()]
|
||||||
|
|
||||||
|
self.check_type_definition(value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class SettingsDirectoryValue(SettingsValue):
|
||||||
|
"""Check and update a setting value that is a directory path
|
||||||
|
"""
|
||||||
|
|
||||||
|
def check_type_definition(self, value: typing.Any) -> typing.Any:
|
||||||
|
super().check_type_definition(value)
|
||||||
|
if not os.path.isdir(value):
|
||||||
|
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), value)
|
||||||
|
|
||||||
|
def __call__(self, value: typing.Any) -> typing.Any:
|
||||||
|
if value == '':
|
||||||
|
value = self.default
|
||||||
|
return super().__call__(value)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_schema(settings, schema, path_list):
|
||||||
|
error = False
|
||||||
|
for key, value in schema.items():
|
||||||
|
if isinstance(value, SettingsValue):
|
||||||
|
try:
|
||||||
|
settings[key] = value(settings.get(key, _UNDEFINED))
|
||||||
|
except Exception as e: # pylint: disable=broad-except
|
||||||
|
# don't stop now: check other values
|
||||||
|
logger.error('%s: %s', '.'.join([*path_list, key]), e)
|
||||||
|
error = True
|
||||||
|
elif isinstance(value, dict):
|
||||||
|
error = error or apply_schema(settings.setdefault(key, {}), schema[key], [*path_list, key])
|
||||||
|
else:
|
||||||
|
settings.setdefault(key, value)
|
||||||
|
if len(path_list) == 0 and error:
|
||||||
|
raise ValueError('Invalid settings.yml')
|
||||||
|
return error
|
||||||
|
|
||||||
|
|
||||||
|
SCHEMA = {
|
||||||
|
'general': {
|
||||||
|
'debug': SettingsValue(bool, False, 'SEARX_DEBUG'),
|
||||||
|
'instance_name': SettingsValue(str, 'searxng'),
|
||||||
|
'contact_url': SettingsValue((None, False, str), None),
|
||||||
|
},
|
||||||
|
'brand': {
|
||||||
|
},
|
||||||
|
'search': {
|
||||||
|
'safe_search': SettingsValue((0,1,2), 0),
|
||||||
|
'autocomplete': SettingsValue(str, ''),
|
||||||
|
'default_lang': SettingsValue(LANGUAGE_CODES, ''),
|
||||||
|
'ban_time_on_fail': SettingsValue(numbers.Real, 5),
|
||||||
|
'max_ban_time_on_fail': SettingsValue(numbers.Real, 120),
|
||||||
|
'formats': SettingsValue(list, OUTPUT_FORMATS),
|
||||||
|
},
|
||||||
|
'server': {
|
||||||
|
'port': SettingsValue(int, 8888),
|
||||||
|
'bind_address': SettingsValue(str, '127.0.0.1', 'SEARX_BIND_ADDRESS'),
|
||||||
|
'secret_key': SettingsValue(str, environ_name='SEARX_SECRET'),
|
||||||
|
'base_url': SettingsValue((False, str), False),
|
||||||
|
'image_proxy': SettingsValue(bool, False),
|
||||||
|
'http_protocol_version': SettingsValue(('1.0', '1.1'), '1.0'),
|
||||||
|
'method': SettingsValue(('POST', 'GET'), 'POST'),
|
||||||
|
'default_http_headers': SettingsValue(dict, {}),
|
||||||
|
},
|
||||||
|
'ui': {
|
||||||
|
'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')),
|
||||||
|
'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')),
|
||||||
|
'default_theme': SettingsValue(str, 'oscar'),
|
||||||
|
'default_locale': SettingsValue(str, ''),
|
||||||
|
'theme_args': {
|
||||||
|
'oscar_style': SettingsValue(OSCAR_STYLE, 'logicodev'),
|
||||||
|
},
|
||||||
|
'results_on_new_tab': SettingsValue(bool, False),
|
||||||
|
'advanced_search': SettingsValue(bool, False),
|
||||||
|
'categories_order': SettingsValue(list, CATEGORY_ORDER),
|
||||||
|
},
|
||||||
|
'preferences': {
|
||||||
|
'lock': SettingsValue(list, []),
|
||||||
|
},
|
||||||
|
'outgoing': {
|
||||||
|
'useragent_suffix': SettingsValue(str, ''),
|
||||||
|
'request_timeout': SettingsValue(numbers.Real, 3.0),
|
||||||
|
'enable_http2': SettingsValue(bool, True),
|
||||||
|
'max_request_timeout': SettingsValue((None, numbers.Real), None),
|
||||||
|
# Magic number kept from previous code
|
||||||
|
'pool_connections': SettingsValue(int, 100),
|
||||||
|
# Picked from constructor
|
||||||
|
'pool_maxsize': SettingsValue(int, 10),
|
||||||
|
'keepalive_expiry': SettingsValue(numbers.Real, 5.0),
|
||||||
|
# default maximum redirect
|
||||||
|
# from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55
|
||||||
|
'max_redirects': SettingsValue(int, 30),
|
||||||
|
'retries': SettingsValue(int, 0),
|
||||||
|
'proxies': SettingsValue((None, str, dict), None),
|
||||||
|
'source_ips': SettingsValue((None, str, list), None),
|
||||||
|
# Tor configuration
|
||||||
|
'using_tor_proxy': SettingsValue(bool, False),
|
||||||
|
'extra_proxy_timeout': SettingsValue(int, 0),
|
||||||
|
'networks': {
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'plugins': SettingsValue((None, list), None),
|
||||||
|
'enabled_plugins': SettingsValue(list, []),
|
||||||
|
'checker': {
|
||||||
|
'off_when_debug': SettingsValue(bool, True),
|
||||||
|
},
|
||||||
|
'engines': SettingsValue(list, []),
|
||||||
|
'locales': SettingsValue(dict, {'en': 'English'}),
|
||||||
|
'doi_resolvers': {
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def settings_set_defaults(settings):
|
||||||
|
apply_schema(settings, SCHEMA, [])
|
||||||
|
return settings
|
@ -8,7 +8,6 @@ from os.path import splitext, join
|
|||||||
from random import choice
|
from random import choice
|
||||||
from html.parser import HTMLParser
|
from html.parser import HTMLParser
|
||||||
from urllib.parse import urljoin, urlparse
|
from urllib.parse import urljoin, urlparse
|
||||||
from collections.abc import Mapping
|
|
||||||
|
|
||||||
from lxml import html
|
from lxml import html
|
||||||
from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult
|
from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult
|
||||||
@ -46,7 +45,7 @@ def searx_useragent():
|
|||||||
"""Return the searx User Agent"""
|
"""Return the searx User Agent"""
|
||||||
return 'searx/{searx_version} {suffix}'.format(
|
return 'searx/{searx_version} {suffix}'.format(
|
||||||
searx_version=VERSION_STRING,
|
searx_version=VERSION_STRING,
|
||||||
suffix=settings['outgoing'].get('useragent_suffix', '')).strip()
|
suffix=settings['outgoing']['useragent_suffix'].strip())
|
||||||
|
|
||||||
|
|
||||||
def gen_useragent(os=None):
|
def gen_useragent(os=None):
|
||||||
@ -501,58 +500,6 @@ def get_engine_from_settings(name):
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
NOT_EXISTS = object()
|
|
||||||
"""Singleton used by :py:obj:`get_value` if a key does not exists."""
|
|
||||||
|
|
||||||
|
|
||||||
def get_value(dictionary, *keys, default=NOT_EXISTS):
|
|
||||||
"""Return the value from a *deep* mapping type (e.g. the ``settings`` object
|
|
||||||
from yaml). If the path to the *key* does not exists a :py:obj:`NOT_EXISTS`
|
|
||||||
is returned (non ``KeyError`` exception is raised).
|
|
||||||
|
|
||||||
.. code: python
|
|
||||||
|
|
||||||
>>> from searx import settings
|
|
||||||
>>> from searx.utils import get_value, NOT_EXISTS
|
|
||||||
>>> get_value(settings, 'checker', 'additional_tests', 'rosebud', 'result_container')
|
|
||||||
['not_empty', ['one_title_contains', 'citizen kane']]
|
|
||||||
|
|
||||||
>>> get_value(settings, 'search', 'xxx') is NOT_EXISTS
|
|
||||||
True
|
|
||||||
>>> get_value(settings, 'search', 'formats')
|
|
||||||
['html', 'csv', 'json', 'rss']
|
|
||||||
|
|
||||||
The list returned from the ``search.format`` key is not a mapping type, you
|
|
||||||
can't traverse along non-mapping types. If you try it, you will get a
|
|
||||||
:py:ref:`NOT_EXISTS`:
|
|
||||||
|
|
||||||
.. code: python
|
|
||||||
|
|
||||||
>>> get_value(settings, 'search', 'format', 'csv') is NOT_EXISTS
|
|
||||||
True
|
|
||||||
>>> get_value(settings, 'search', 'formats')[1]
|
|
||||||
'csv'
|
|
||||||
|
|
||||||
For convenience you can replace :py:ref:`NOT_EXISTS` by a default value of
|
|
||||||
your choice:
|
|
||||||
|
|
||||||
.. code: python
|
|
||||||
|
|
||||||
if 'csv' in get_value(settings, 'search', 'formats', default=[]):
|
|
||||||
print("csv format is denied")
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
obj = dictionary
|
|
||||||
for k in keys:
|
|
||||||
if not isinstance(obj, Mapping):
|
|
||||||
raise TypeError("expected mapping type, got %s" % type(obj))
|
|
||||||
obj = obj.get(k, default)
|
|
||||||
if obj is default:
|
|
||||||
return obj
|
|
||||||
return obj
|
|
||||||
|
|
||||||
|
|
||||||
def get_xpath(xpath_spec):
|
def get_xpath(xpath_spec):
|
||||||
"""Return cached compiled XPath
|
"""Return cached compiled XPath
|
||||||
|
|
||||||
|
@ -56,12 +56,12 @@ from flask_babel import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from searx import logger
|
from searx import logger
|
||||||
from searx import brand, static_path
|
from searx import brand
|
||||||
from searx import (
|
from searx import (
|
||||||
settings,
|
settings,
|
||||||
searx_dir,
|
|
||||||
searx_debug,
|
searx_debug,
|
||||||
)
|
)
|
||||||
|
from searx.settings_defaults import OUTPUT_FORMATS
|
||||||
from searx.exceptions import SearxParameterException
|
from searx.exceptions import SearxParameterException
|
||||||
from searx.engines import (
|
from searx.engines import (
|
||||||
categories,
|
categories,
|
||||||
@ -71,7 +71,6 @@ from searx.engines import (
|
|||||||
from searx.webutils import (
|
from searx.webutils import (
|
||||||
UnicodeWriter,
|
UnicodeWriter,
|
||||||
highlight_content,
|
highlight_content,
|
||||||
get_resources_directory,
|
|
||||||
get_static_files,
|
get_static_files,
|
||||||
get_result_templates,
|
get_result_templates,
|
||||||
get_themes,
|
get_themes,
|
||||||
@ -88,7 +87,6 @@ from searx.utils import (
|
|||||||
gen_useragent,
|
gen_useragent,
|
||||||
dict_subset,
|
dict_subset,
|
||||||
match_language,
|
match_language,
|
||||||
get_value,
|
|
||||||
)
|
)
|
||||||
from searx.version import VERSION_STRING
|
from searx.version import VERSION_STRING
|
||||||
from searx.query import RawTextQuery
|
from searx.query import RawTextQuery
|
||||||
@ -139,7 +137,7 @@ if sys.version_info[0] < 3:
|
|||||||
logger = logger.getChild('webapp')
|
logger = logger.getChild('webapp')
|
||||||
|
|
||||||
# serve pages with HTTP/1.1
|
# serve pages with HTTP/1.1
|
||||||
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
|
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])
|
||||||
|
|
||||||
# check secret_key
|
# check secret_key
|
||||||
if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
|
if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
|
||||||
@ -147,25 +145,22 @@ if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
# about static
|
# about static
|
||||||
static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
|
logger.debug('static directory is %s', settings['ui']['static_path'])
|
||||||
logger.debug('static directory is %s', static_path)
|
static_files = get_static_files(settings['ui']['static_path'])
|
||||||
static_files = get_static_files(static_path)
|
|
||||||
|
|
||||||
# about templates
|
# about templates
|
||||||
|
logger.debug('templates directory is %s', settings['ui']['templates_path'])
|
||||||
default_theme = settings['ui']['default_theme']
|
default_theme = settings['ui']['default_theme']
|
||||||
templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
|
templates_path = settings['ui']['templates_path']
|
||||||
logger.debug('templates directory is %s', templates_path)
|
|
||||||
themes = get_themes(templates_path)
|
themes = get_themes(templates_path)
|
||||||
result_templates = get_result_templates(templates_path)
|
result_templates = get_result_templates(templates_path)
|
||||||
global_favicons = []
|
global_favicons = []
|
||||||
for indice, theme in enumerate(themes):
|
for indice, theme in enumerate(themes):
|
||||||
global_favicons.append([])
|
global_favicons.append([])
|
||||||
theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
|
theme_img_path = os.path.join(settings['ui']['static_path'], 'themes', theme, 'img', 'icons')
|
||||||
for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
|
for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
|
||||||
global_favicons[indice].extend(filenames)
|
global_favicons[indice].extend(filenames)
|
||||||
|
|
||||||
OUTPUT_FORMATS = ['html', 'csv', 'json', 'rss']
|
|
||||||
|
|
||||||
STATS_SORT_PARAMETERS = {
|
STATS_SORT_PARAMETERS = {
|
||||||
'name': (False, 'name', ''),
|
'name': (False, 'name', ''),
|
||||||
'score': (True, 'score', 0),
|
'score': (True, 'score', 0),
|
||||||
@ -177,7 +172,7 @@ STATS_SORT_PARAMETERS = {
|
|||||||
# Flask app
|
# Flask app
|
||||||
app = Flask(
|
app = Flask(
|
||||||
__name__,
|
__name__,
|
||||||
static_folder=static_path,
|
static_folder=settings['ui']['static_path'],
|
||||||
template_folder=templates_path
|
template_folder=templates_path
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -517,8 +512,7 @@ def render(template_name, override_theme=None, **kwargs):
|
|||||||
kwargs['preferences'] = request.preferences
|
kwargs['preferences'] = request.preferences
|
||||||
|
|
||||||
kwargs['search_formats'] = [
|
kwargs['search_formats'] = [
|
||||||
x for x in get_value(
|
x for x in settings['search']['formats']
|
||||||
settings, 'search', 'formats', default=OUTPUT_FORMATS)
|
|
||||||
if x != 'html']
|
if x != 'html']
|
||||||
|
|
||||||
kwargs['brand'] = brand
|
kwargs['brand'] = brand
|
||||||
@ -545,12 +539,7 @@ def render(template_name, override_theme=None, **kwargs):
|
|||||||
|
|
||||||
|
|
||||||
def _get_ordered_categories():
|
def _get_ordered_categories():
|
||||||
ordered_categories = []
|
ordered_categories = list(settings['ui']['categories_order'])
|
||||||
if 'categories_order' not in settings['ui']:
|
|
||||||
ordered_categories = ['general']
|
|
||||||
ordered_categories.extend(x for x in sorted(categories.keys()) if x != 'general')
|
|
||||||
return ordered_categories
|
|
||||||
ordered_categories = settings['ui']['categories_order']
|
|
||||||
ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
|
ordered_categories.extend(x for x in sorted(categories.keys()) if x not in ordered_categories)
|
||||||
return ordered_categories
|
return ordered_categories
|
||||||
|
|
||||||
@ -610,7 +599,7 @@ def pre_request():
|
|||||||
@app.after_request
|
@app.after_request
|
||||||
def add_default_headers(response):
|
def add_default_headers(response):
|
||||||
# set default http headers
|
# set default http headers
|
||||||
for header, value in settings['server'].get('default_http_headers', {}).items():
|
for header, value in settings['server']['default_http_headers'].items():
|
||||||
if header in response.headers:
|
if header in response.headers:
|
||||||
continue
|
continue
|
||||||
response.headers[header] = value
|
response.headers[header] = value
|
||||||
@ -696,7 +685,7 @@ def search():
|
|||||||
if output_format not in OUTPUT_FORMATS:
|
if output_format not in OUTPUT_FORMATS:
|
||||||
output_format = 'html'
|
output_format = 'html'
|
||||||
|
|
||||||
if output_format not in get_value(settings, 'search', 'formats', default=OUTPUT_FORMATS):
|
if output_format not in settings['search']['formats']:
|
||||||
flask.abort(403)
|
flask.abort(403)
|
||||||
|
|
||||||
# check if there is query (not None and not an empty string)
|
# check if there is query (not None and not an empty string)
|
||||||
@ -1069,11 +1058,6 @@ def preferences():
|
|||||||
'time_range_support': time_range_support,
|
'time_range_support': time_range_support,
|
||||||
}
|
}
|
||||||
|
|
||||||
#
|
|
||||||
locked_preferences = list()
|
|
||||||
if 'preferences' in settings and 'lock' in settings['preferences']:
|
|
||||||
locked_preferences = settings['preferences']['lock']
|
|
||||||
|
|
||||||
#
|
#
|
||||||
return render('preferences.html',
|
return render('preferences.html',
|
||||||
selected_categories=get_selected_categories(request.preferences, request.form),
|
selected_categories=get_selected_categories(request.preferences, request.form),
|
||||||
@ -1098,7 +1082,7 @@ def preferences():
|
|||||||
theme=get_current_theme_name(),
|
theme=get_current_theme_name(),
|
||||||
preferences_url_params=request.preferences.get_as_url_params(),
|
preferences_url_params=request.preferences.get_as_url_params(),
|
||||||
base_url=get_base_url(),
|
base_url=get_base_url(),
|
||||||
locked_preferences=locked_preferences,
|
locked_preferences=settings['preferences']['lock'],
|
||||||
preferences=True)
|
preferences=True)
|
||||||
|
|
||||||
|
|
||||||
@ -1271,7 +1255,7 @@ def favicon():
|
|||||||
return send_from_directory(
|
return send_from_directory(
|
||||||
os.path.join(
|
os.path.join(
|
||||||
app.root_path,
|
app.root_path,
|
||||||
static_path,
|
settings['ui']['static_path'],
|
||||||
'themes',
|
'themes',
|
||||||
get_current_theme_name(),
|
get_current_theme_name(),
|
||||||
'img'),
|
'img'),
|
||||||
|
@ -47,14 +47,6 @@ class UnicodeWriter:
|
|||||||
self.writerow(row)
|
self.writerow(row)
|
||||||
|
|
||||||
|
|
||||||
def get_resources_directory(searx_directory, subdirectory, resources_directory):
|
|
||||||
if not resources_directory:
|
|
||||||
resources_directory = os.path.join(searx_directory, subdirectory)
|
|
||||||
if not os.path.isdir(resources_directory):
|
|
||||||
raise Exception(resources_directory + " is not a directory")
|
|
||||||
return resources_directory
|
|
||||||
|
|
||||||
|
|
||||||
def get_themes(templates_path):
|
def get_themes(templates_path):
|
||||||
"""Returns available themes list."""
|
"""Returns available themes list."""
|
||||||
themes = os.listdir(templates_path)
|
themes = os.listdir(templates_path)
|
||||||
|
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from searx.testing import SearxTestCase
|
from searx.testing import SearxTestCase
|
||||||
from searx.search import SearchQuery, EngineRef
|
from searx.search import SearchQuery, EngineRef
|
||||||
|
from searx import settings
|
||||||
import searx.search
|
import searx.search
|
||||||
|
|
||||||
|
|
||||||
@ -41,7 +42,7 @@ class SearchTestCase(SearxTestCase):
|
|||||||
searx.search.initialize(TEST_ENGINES)
|
searx.search.initialize(TEST_ENGINES)
|
||||||
|
|
||||||
def test_timeout_simple(self):
|
def test_timeout_simple(self):
|
||||||
searx.search.max_request_timeout = None
|
settings['outgoing']['max_request_timeout'] = None
|
||||||
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
||||||
'en-US', SAFESEARCH, PAGENO, None, None)
|
'en-US', SAFESEARCH, PAGENO, None, None)
|
||||||
search = searx.search.Search(search_query)
|
search = searx.search.Search(search_query)
|
||||||
@ -49,7 +50,7 @@ class SearchTestCase(SearxTestCase):
|
|||||||
self.assertEqual(search.actual_timeout, 3.0)
|
self.assertEqual(search.actual_timeout, 3.0)
|
||||||
|
|
||||||
def test_timeout_query_above_default_nomax(self):
|
def test_timeout_query_above_default_nomax(self):
|
||||||
searx.search.max_request_timeout = None
|
settings['outgoing']['max_request_timeout'] = None
|
||||||
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
||||||
'en-US', SAFESEARCH, PAGENO, None, 5.0)
|
'en-US', SAFESEARCH, PAGENO, None, 5.0)
|
||||||
search = searx.search.Search(search_query)
|
search = searx.search.Search(search_query)
|
||||||
@ -57,7 +58,7 @@ class SearchTestCase(SearxTestCase):
|
|||||||
self.assertEqual(search.actual_timeout, 3.0)
|
self.assertEqual(search.actual_timeout, 3.0)
|
||||||
|
|
||||||
def test_timeout_query_below_default_nomax(self):
|
def test_timeout_query_below_default_nomax(self):
|
||||||
searx.search.max_request_timeout = None
|
settings['outgoing']['max_request_timeout'] = None
|
||||||
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
||||||
'en-US', SAFESEARCH, PAGENO, None, 1.0)
|
'en-US', SAFESEARCH, PAGENO, None, 1.0)
|
||||||
search = searx.search.Search(search_query)
|
search = searx.search.Search(search_query)
|
||||||
@ -65,7 +66,7 @@ class SearchTestCase(SearxTestCase):
|
|||||||
self.assertEqual(search.actual_timeout, 1.0)
|
self.assertEqual(search.actual_timeout, 1.0)
|
||||||
|
|
||||||
def test_timeout_query_below_max(self):
|
def test_timeout_query_below_max(self):
|
||||||
searx.search.max_request_timeout = 10.0
|
settings['outgoing']['max_request_timeout'] = 10.0
|
||||||
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
||||||
'en-US', SAFESEARCH, PAGENO, None, 5.0)
|
'en-US', SAFESEARCH, PAGENO, None, 5.0)
|
||||||
search = searx.search.Search(search_query)
|
search = searx.search.Search(search_query)
|
||||||
@ -73,7 +74,7 @@ class SearchTestCase(SearxTestCase):
|
|||||||
self.assertEqual(search.actual_timeout, 5.0)
|
self.assertEqual(search.actual_timeout, 5.0)
|
||||||
|
|
||||||
def test_timeout_query_above_max(self):
|
def test_timeout_query_above_max(self):
|
||||||
searx.search.max_request_timeout = 10.0
|
settings['outgoing']['max_request_timeout'] = 10.0
|
||||||
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],
|
||||||
'en-US', SAFESEARCH, PAGENO, None, 15.0)
|
'en-US', SAFESEARCH, PAGENO, None, 15.0)
|
||||||
search = searx.search.Search(search_query)
|
search = searx.search.Search(search_query)
|
||||||
|
Loading…
Reference in New Issue
Block a user