2020-12-16 13:41:32 +01:00
|
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
|
|
|
|
from time import time
|
2021-03-18 19:59:01 +01:00
|
|
|
import asyncio
|
2020-12-16 13:41:32 +01:00
|
|
|
|
2021-03-18 19:59:01 +01:00
|
|
|
import httpx
|
2020-12-16 13:41:32 +01:00
|
|
|
|
[httpx] replace searx.poolrequests by searx.network
settings.yml:
* outgoing.networks:
* can contains network definition
* propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
* retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
* local_addresses can be "192.168.0.1/24" (it supports IPv6)
* support_ipv4 & support_ipv6: both True by default
see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
* either a full network description
* either reference an existing network
* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)
2021-04-05 10:43:33 +02:00
|
|
|
import searx.network
|
2020-12-16 13:41:32 +01:00
|
|
|
from searx import logger
|
|
|
|
from searx.utils import gen_useragent
|
|
|
|
from searx.exceptions import (SearxEngineAccessDeniedException, SearxEngineCaptchaException,
|
|
|
|
SearxEngineTooManyRequestsException,)
|
2021-04-14 17:23:15 +02:00
|
|
|
from searx.metrics.error_recorder import count_error
|
2020-12-16 13:41:32 +01:00
|
|
|
|
|
|
|
from searx.search.processors.abstract import EngineProcessor
|
|
|
|
|
|
|
|
|
2021-04-13 15:21:53 +02:00
|
|
|
logger = logger.getChild('searx.search.processor.online')
|
2020-12-16 13:41:32 +01:00
|
|
|
|
2020-12-17 16:49:48 +01:00
|
|
|
|
|
|
|
def default_request_params():
|
|
|
|
return {
|
|
|
|
'method': 'GET',
|
|
|
|
'headers': {},
|
|
|
|
'data': {},
|
|
|
|
'url': '',
|
|
|
|
'cookies': {},
|
|
|
|
'verify': True,
|
|
|
|
'auth': None
|
|
|
|
}
|
2020-12-16 13:41:32 +01:00
|
|
|
|
|
|
|
|
|
|
|
class OnlineProcessor(EngineProcessor):
|
|
|
|
|
|
|
|
engine_type = 'online'
|
|
|
|
|
|
|
|
def get_params(self, search_query, engine_category):
|
|
|
|
params = super().get_params(search_query, engine_category)
|
|
|
|
if params is None:
|
|
|
|
return None
|
|
|
|
|
|
|
|
# add default params
|
2020-12-17 16:49:48 +01:00
|
|
|
params.update(default_request_params())
|
2020-12-16 13:41:32 +01:00
|
|
|
|
|
|
|
# add an user agent
|
|
|
|
params['headers']['User-Agent'] = gen_useragent()
|
|
|
|
|
|
|
|
return params
|
|
|
|
|
|
|
|
def _send_http_request(self, params):
|
|
|
|
# create dictionary which contain all
|
|
|
|
# informations about the request
|
|
|
|
request_args = dict(
|
|
|
|
headers=params['headers'],
|
|
|
|
cookies=params['cookies'],
|
|
|
|
verify=params['verify'],
|
|
|
|
auth=params['auth']
|
|
|
|
)
|
|
|
|
|
|
|
|
# max_redirects
|
|
|
|
max_redirects = params.get('max_redirects')
|
|
|
|
if max_redirects:
|
|
|
|
request_args['max_redirects'] = max_redirects
|
|
|
|
|
2021-02-09 12:07:19 +01:00
|
|
|
# allow_redirects
|
|
|
|
if 'allow_redirects' in params:
|
|
|
|
request_args['allow_redirects'] = params['allow_redirects']
|
|
|
|
|
2020-12-16 13:41:32 +01:00
|
|
|
# soft_max_redirects
|
|
|
|
soft_max_redirects = params.get('soft_max_redirects', max_redirects or 0)
|
|
|
|
|
|
|
|
# raise_for_status
|
2021-02-09 11:27:41 +01:00
|
|
|
request_args['raise_for_httperror'] = params.get('raise_for_httperror', True)
|
2020-12-16 13:41:32 +01:00
|
|
|
|
|
|
|
# specific type of request (GET or POST)
|
|
|
|
if params['method'] == 'GET':
|
[httpx] replace searx.poolrequests by searx.network
settings.yml:
* outgoing.networks:
* can contains network definition
* propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
* retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
* local_addresses can be "192.168.0.1/24" (it supports IPv6)
* support_ipv4 & support_ipv6: both True by default
see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
* either a full network description
* either reference an existing network
* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)
2021-04-05 10:43:33 +02:00
|
|
|
req = searx.network.get
|
2020-12-16 13:41:32 +01:00
|
|
|
else:
|
[httpx] replace searx.poolrequests by searx.network
settings.yml:
* outgoing.networks:
* can contains network definition
* propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
* retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
* local_addresses can be "192.168.0.1/24" (it supports IPv6)
* support_ipv4 & support_ipv6: both True by default
see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
* either a full network description
* either reference an existing network
* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)
2021-04-05 10:43:33 +02:00
|
|
|
req = searx.network.post
|
2020-12-16 13:41:32 +01:00
|
|
|
|
|
|
|
request_args['data'] = params['data']
|
|
|
|
|
|
|
|
# send the request
|
|
|
|
response = req(params['url'], **request_args)
|
|
|
|
|
|
|
|
# check soft limit of the redirect count
|
|
|
|
if len(response.history) > soft_max_redirects:
|
|
|
|
# unexpected redirect : record an error
|
|
|
|
# but the engine might still return valid results.
|
|
|
|
status_code = str(response.status_code or '')
|
2021-03-18 19:59:01 +01:00
|
|
|
reason = response.reason_phrase or ''
|
|
|
|
hostname = response.url.host
|
2021-04-14 17:23:15 +02:00
|
|
|
count_error(self.engine_name,
|
|
|
|
'{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects),
|
|
|
|
(status_code, reason, hostname))
|
2020-12-16 13:41:32 +01:00
|
|
|
|
|
|
|
return response
|
|
|
|
|
|
|
|
def _search_basic(self, query, params):
|
|
|
|
# update request parameters dependent on
|
|
|
|
# search-engine (contained in engines folder)
|
|
|
|
self.engine.request(query, params)
|
|
|
|
|
|
|
|
# ignoring empty urls
|
|
|
|
if params['url'] is None:
|
|
|
|
return None
|
|
|
|
|
|
|
|
if not params['url']:
|
|
|
|
return None
|
|
|
|
|
|
|
|
# send request
|
|
|
|
response = self._send_http_request(params)
|
|
|
|
|
|
|
|
# parse the response
|
|
|
|
response.search_params = params
|
|
|
|
return self.engine.response(response)
|
|
|
|
|
|
|
|
def search(self, query, params, result_container, start_time, timeout_limit):
|
|
|
|
# set timeout for all HTTP requests
|
[httpx] replace searx.poolrequests by searx.network
settings.yml:
* outgoing.networks:
* can contains network definition
* propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
* retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
* local_addresses can be "192.168.0.1/24" (it supports IPv6)
* support_ipv4 & support_ipv6: both True by default
see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
* either a full network description
* either reference an existing network
* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)
2021-04-05 10:43:33 +02:00
|
|
|
searx.network.set_timeout_for_thread(timeout_limit, start_time=start_time)
|
2020-12-16 13:41:32 +01:00
|
|
|
# reset the HTTP total time
|
[httpx] replace searx.poolrequests by searx.network
settings.yml:
* outgoing.networks:
* can contains network definition
* propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
* retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
* local_addresses can be "192.168.0.1/24" (it supports IPv6)
* support_ipv4 & support_ipv6: both True by default
see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
* either a full network description
* either reference an existing network
* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)
2021-04-05 10:43:33 +02:00
|
|
|
searx.network.reset_time_for_thread()
|
|
|
|
# set the network
|
|
|
|
searx.network.set_context_network_name(self.engine_name)
|
2020-12-16 13:41:32 +01:00
|
|
|
|
|
|
|
try:
|
|
|
|
# send requests and parse the results
|
|
|
|
search_results = self._search_basic(query, params)
|
2021-04-13 15:21:53 +02:00
|
|
|
self.extend_container(result_container, start_time, search_results)
|
|
|
|
except (httpx.TimeoutException, asyncio.TimeoutError) as e:
|
|
|
|
# requests timeout (connect or read)
|
|
|
|
self.handle_exception(result_container, 'HTTP timeout', e, suspend=True, display_exception=False)
|
|
|
|
logger.error("engine {0} : HTTP requests timeout"
|
|
|
|
"(search duration : {1} s, timeout: {2} s) : {3}"
|
|
|
|
.format(self.engine_name, time() - start_time,
|
|
|
|
timeout_limit,
|
|
|
|
e.__class__.__name__))
|
|
|
|
except (httpx.HTTPError, httpx.StreamError) as e:
|
|
|
|
# other requests exception
|
|
|
|
self.handle_exception(result_container, 'HTTP error', e, suspend=True, display_exception=False)
|
|
|
|
logger.exception("engine {0} : requests exception"
|
2020-12-16 13:41:32 +01:00
|
|
|
"(search duration : {1} s, timeout: {2} s) : {3}"
|
2021-04-13 15:21:53 +02:00
|
|
|
.format(self.engine_name, time() - start_time,
|
|
|
|
timeout_limit,
|
|
|
|
e))
|
|
|
|
except SearxEngineCaptchaException as e:
|
|
|
|
self.handle_exception(result_container, 'CAPTCHA required', e, suspend=True, display_exception=False)
|
|
|
|
logger.exception('engine {0} : CAPTCHA'.format(self.engine_name))
|
|
|
|
except SearxEngineTooManyRequestsException as e:
|
|
|
|
self.handle_exception(result_container, 'too many requests', e, suspend=True, display_exception=False)
|
|
|
|
logger.exception('engine {0} : Too many requests'.format(self.engine_name))
|
|
|
|
except SearxEngineAccessDeniedException as e:
|
|
|
|
self.handle_exception(result_container, 'blocked', e, suspend=True, display_exception=False)
|
|
|
|
logger.exception('engine {0} : Searx is blocked'.format(self.engine_name))
|
|
|
|
except Exception as e:
|
|
|
|
self.handle_exception(result_container, 'unexpected crash', e, display_exception=False)
|
|
|
|
logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e))
|
2020-12-24 09:28:16 +01:00
|
|
|
|
|
|
|
def get_default_tests(self):
|
|
|
|
tests = {}
|
|
|
|
|
|
|
|
tests['simple'] = {
|
2021-01-08 19:05:56 +01:00
|
|
|
'matrix': {'query': ('life', 'computer')},
|
2020-12-24 09:28:16 +01:00
|
|
|
'result_container': ['not_empty'],
|
|
|
|
}
|
|
|
|
|
|
|
|
if getattr(self.engine, 'paging', False):
|
|
|
|
tests['paging'] = {
|
|
|
|
'matrix': {'query': 'time',
|
|
|
|
'pageno': (1, 2, 3)},
|
|
|
|
'result_container': ['not_empty'],
|
|
|
|
'test': ['unique_results']
|
|
|
|
}
|
2021-01-08 19:05:56 +01:00
|
|
|
if 'general' in self.engine.categories:
|
|
|
|
# avoid documentation about HTML tags (<time> and <input type="time">)
|
|
|
|
tests['paging']['matrix']['query'] = 'news'
|
2020-12-24 09:28:16 +01:00
|
|
|
|
|
|
|
if getattr(self.engine, 'time_range', False):
|
|
|
|
tests['time_range'] = {
|
2021-01-08 19:05:56 +01:00
|
|
|
'matrix': {'query': 'news',
|
2020-12-24 09:28:16 +01:00
|
|
|
'time_range': (None, 'day')},
|
|
|
|
'result_container': ['not_empty'],
|
|
|
|
'test': ['unique_results']
|
|
|
|
}
|
|
|
|
|
2021-01-19 21:29:31 +01:00
|
|
|
if getattr(self.engine, 'supported_languages', []):
|
2020-12-24 09:28:16 +01:00
|
|
|
tests['lang_fr'] = {
|
|
|
|
'matrix': {'query': 'paris', 'lang': 'fr'},
|
2021-01-19 21:29:31 +01:00
|
|
|
'result_container': ['not_empty', ('has_language', 'fr')],
|
2020-12-24 09:28:16 +01:00
|
|
|
}
|
|
|
|
tests['lang_en'] = {
|
|
|
|
'matrix': {'query': 'paris', 'lang': 'en'},
|
2021-01-19 21:29:31 +01:00
|
|
|
'result_container': ['not_empty', ('has_language', 'en')],
|
2020-12-24 09:28:16 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if getattr(self.engine, 'safesearch', False):
|
|
|
|
tests['safesearch'] = {
|
|
|
|
'matrix': {'query': 'porn',
|
|
|
|
'safesearch': (0, 2)},
|
|
|
|
'test': ['unique_results']
|
|
|
|
}
|
|
|
|
|
|
|
|
return tests
|