mirror of
https://github.com/searxng/searxng.git
synced 2024-08-24 23:13:33 +02:00
Merge branch 'master' of https://github.com/asciimoo/searx into filtron
This commit is contained in:
commit
1fdd8cd3c0
@ -1,12 +1,12 @@
|
||||
{
|
||||
"versions": [
|
||||
"74.0",
|
||||
"73.0.1",
|
||||
"73.0"
|
||||
"75.0",
|
||||
"74.0.1",
|
||||
"74.0"
|
||||
],
|
||||
"os": [
|
||||
"Windows NT 10; WOW64",
|
||||
"Windows NT 10.0; WOW64",
|
||||
"X11; Linux x86_64"
|
||||
],
|
||||
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
|
||||
}
|
||||
}
|
||||
|
@ -1,96 +0,0 @@
|
||||
"""
|
||||
Faroo (Web, News)
|
||||
|
||||
@website http://www.faroo.com
|
||||
@provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
|
||||
|
||||
@using-api no
|
||||
@results JSON
|
||||
@stable yes
|
||||
@parse url, title, content, publishedDate, img_src
|
||||
"""
|
||||
|
||||
from json import loads
|
||||
import datetime
|
||||
from searx.utils import searx_useragent
|
||||
from searx.url_utils import urlencode
|
||||
|
||||
# engine dependent config
|
||||
categories = ['general', 'news']
|
||||
paging = True
|
||||
language_support = True
|
||||
number_of_results = 10
|
||||
|
||||
# search-url
|
||||
url = 'http://www.faroo.com/'
|
||||
search_url = url + 'instant.json?{query}'\
|
||||
'&start={offset}'\
|
||||
'&length={number_of_results}'\
|
||||
'&l={language}'\
|
||||
'&src={categorie}'\
|
||||
'&i=false'\
|
||||
'&c=false'
|
||||
|
||||
search_category = {'general': 'web',
|
||||
'news': 'news'}
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
offset = (params['pageno'] - 1) * number_of_results + 1
|
||||
categorie = search_category.get(params['category'], 'web')
|
||||
|
||||
if params['language'] == 'all':
|
||||
language = 'en'
|
||||
else:
|
||||
language = params['language'].split('-')[0]
|
||||
|
||||
# if language is not supported, put it in english
|
||||
if language != 'en' and\
|
||||
language != 'de' and\
|
||||
language != 'zh':
|
||||
language = 'en'
|
||||
|
||||
params['url'] = search_url.format(offset=offset,
|
||||
number_of_results=number_of_results,
|
||||
query=urlencode({'q': query}),
|
||||
language=language,
|
||||
categorie=categorie)
|
||||
|
||||
params['headers']['Referer'] = url
|
||||
|
||||
return params
|
||||
|
||||
|
||||
# get response from search-request
|
||||
def response(resp):
|
||||
# HTTP-Code 429: rate limit exceeded
|
||||
if resp.status_code == 429:
|
||||
raise Exception("rate limit has been exceeded!")
|
||||
|
||||
results = []
|
||||
|
||||
search_res = loads(resp.text)
|
||||
|
||||
# return empty array if there are no results
|
||||
if not search_res.get('results', {}):
|
||||
return []
|
||||
|
||||
# parse results
|
||||
for result in search_res['results']:
|
||||
publishedDate = None
|
||||
result_json = {'url': result['url'], 'title': result['title'],
|
||||
'content': result['kwic']}
|
||||
if result['news']:
|
||||
result_json['publishedDate'] = \
|
||||
datetime.datetime.fromtimestamp(result['date'] / 1000.0)
|
||||
|
||||
# append image result if image url is set
|
||||
if result['iurl']:
|
||||
result_json['template'] = 'videos.html'
|
||||
result_json['thumbnail'] = result['iurl']
|
||||
|
||||
results.append(result_json)
|
||||
|
||||
# return results
|
||||
return results
|
@ -345,8 +345,8 @@ class ResultContainer(object):
|
||||
return 0
|
||||
return resultnum_sum / len(self._number_of_results)
|
||||
|
||||
def add_unresponsive_engine(self, engine_error):
|
||||
self.unresponsive_engines.add(engine_error)
|
||||
def add_unresponsive_engine(self, engine_name, error_type, error_message=None):
|
||||
self.unresponsive_engines.add((engine_name, error_type, error_message))
|
||||
|
||||
def add_timing(self, engine_name, engine_time, page_load_time):
|
||||
self.timings.append({
|
||||
|
@ -127,11 +127,7 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c
|
||||
logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))
|
||||
except Exception as e:
|
||||
record_offline_engine_stats_on_error(engine, result_container, start_time)
|
||||
|
||||
result_container.add_unresponsive_engine((
|
||||
engine_name,
|
||||
u'{0}: {1}'.format(gettext('unexpected crash'), e),
|
||||
))
|
||||
result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
|
||||
logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
|
||||
|
||||
|
||||
@ -186,24 +182,21 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont
|
||||
engine.stats['errors'] += 1
|
||||
|
||||
if (issubclass(e.__class__, requests.exceptions.Timeout)):
|
||||
result_container.add_unresponsive_engine((engine_name, gettext('timeout')))
|
||||
result_container.add_unresponsive_engine(engine_name, 'timeout')
|
||||
# requests timeout (connect or read)
|
||||
logger.error("engine {0} : HTTP requests timeout"
|
||||
"(search duration : {1} s, timeout: {2} s) : {3}"
|
||||
.format(engine_name, engine_time, timeout_limit, e.__class__.__name__))
|
||||
requests_exception = True
|
||||
elif (issubclass(e.__class__, requests.exceptions.RequestException)):
|
||||
result_container.add_unresponsive_engine((engine_name, gettext('request exception')))
|
||||
result_container.add_unresponsive_engine(engine_name, 'request exception')
|
||||
# other requests exception
|
||||
logger.exception("engine {0} : requests exception"
|
||||
"(search duration : {1} s, timeout: {2} s) : {3}"
|
||||
.format(engine_name, engine_time, timeout_limit, e))
|
||||
requests_exception = True
|
||||
else:
|
||||
result_container.add_unresponsive_engine((
|
||||
engine_name,
|
||||
u'{0}: {1}'.format(gettext('unexpected crash'), e),
|
||||
))
|
||||
result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
|
||||
# others errors
|
||||
logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
|
||||
|
||||
@ -238,7 +231,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim
|
||||
remaining_time = max(0.0, timeout_limit - (time() - start_time))
|
||||
th.join(remaining_time)
|
||||
if th.isAlive():
|
||||
result_container.add_unresponsive_engine((th._engine_name, gettext('timeout')))
|
||||
result_container.add_unresponsive_engine(th._engine_name, 'timeout')
|
||||
logger.warning('engine timeout: {0}'.format(th._engine_name))
|
||||
|
||||
|
||||
|
@ -219,11 +219,6 @@ engines:
|
||||
shortcut : et
|
||||
disabled : True
|
||||
|
||||
- name : faroo
|
||||
engine : faroo
|
||||
shortcut : fa
|
||||
disabled : True
|
||||
|
||||
- name : 1x
|
||||
engine : www1x
|
||||
shortcut : 1x
|
||||
|
@ -56,6 +56,7 @@ from flask import (
|
||||
from babel.support import Translations
|
||||
import flask_babel
|
||||
from flask_babel import Babel, gettext, format_date, format_decimal
|
||||
from flask.ctx import has_request_context
|
||||
from flask.json import jsonify
|
||||
from searx import brand
|
||||
from searx import settings, searx_dir, searx_debug
|
||||
@ -165,13 +166,11 @@ _flask_babel_get_translations = flask_babel.get_translations
|
||||
|
||||
# monkey patch for flask_babel.get_translations
|
||||
def _get_translations():
|
||||
translation_locale = request.form.get('use-translation')
|
||||
if translation_locale:
|
||||
if has_request_context() and request.form.get('use-translation') == 'oc':
|
||||
babel_ext = flask_babel.current_app.extensions['babel']
|
||||
translation = Translations.load(next(babel_ext.translation_directories), 'oc')
|
||||
else:
|
||||
translation = _flask_babel_get_translations()
|
||||
return translation
|
||||
return Translations.load(next(babel_ext.translation_directories), 'oc')
|
||||
|
||||
return _flask_babel_get_translations()
|
||||
|
||||
|
||||
flask_babel.get_translations = _get_translations
|
||||
@ -627,7 +626,7 @@ def index():
|
||||
'corrections': list(result_container.corrections),
|
||||
'infoboxes': result_container.infoboxes,
|
||||
'suggestions': list(result_container.suggestions),
|
||||
'unresponsive_engines': list(result_container.unresponsive_engines)},
|
||||
'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)}, # noqa
|
||||
default=lambda item: list(item) if isinstance(item, set) else item),
|
||||
mimetype='application/json')
|
||||
elif output_format == 'csv':
|
||||
@ -695,7 +694,7 @@ def index():
|
||||
corrections=correction_urls,
|
||||
infoboxes=result_container.infoboxes,
|
||||
paging=result_container.paging,
|
||||
unresponsive_engines=result_container.unresponsive_engines,
|
||||
unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),
|
||||
current_language=match_language(search_query.lang,
|
||||
LANGUAGE_CODES,
|
||||
fallback=request.preferences.get_value("language")),
|
||||
@ -706,6 +705,16 @@ def index():
|
||||
)
|
||||
|
||||
|
||||
def __get_translated_errors(unresponsive_engines):
|
||||
translated_errors = []
|
||||
for unresponsive_engine in unresponsive_engines:
|
||||
error_msg = gettext(unresponsive_engine[1])
|
||||
if unresponsive_engine[2]:
|
||||
error_msg = "{} {}".format(error_msg, unresponsive_engine[2])
|
||||
translated_errors.append((unresponsive_engine[0], error_msg))
|
||||
return translated_errors
|
||||
|
||||
|
||||
@app.route('/about', methods=['GET'])
|
||||
def about():
|
||||
"""Render about page"""
|
||||
|
Loading…
Reference in New Issue
Block a user