kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-22 04:01:40 +01:00
Code

fix: robot fw, entry points, some flake8, package searx egg

Browse Source
This commit is contained in:
Matej Cotman 2014-01-19 22:59:01 +01:00
parent b7fa79081f
commit dd4662978d
17 changed files with 293 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.gitignore vendored
View File

@ -1,20 +1,24 @@
env
engines.cfg
.installed.cfg
.coverage
coverage/
.installed.cfg
engines.cfg
env
robot_log.html
robot_output.xml
robot_report.html
setup.cfg
*.pyc
*/*.pyc
bin/
build/
covearge/
develop-eggs/
dist/
eggs/
include/
lib/
build/
develop-eggs/
eggs/
local/
searx.egg-info/
parts/
searx.egg-info/
var/

12
Makefile
View File

@ -21,11 +21,7 @@ $(python):
tests: .installed.cfg
@bin/test
enginescfg:
@test -f ./engines.cfg || echo "Copying engines.cfg ..."
@cp --no-clobber engines.cfg_sample engines.cfg
robot: .installed.cfg enginescfg
robot: .installed.cfg
@bin/robot
flake8: .installed.cfg
@ -37,18 +33,18 @@ coverage: .installed.cfg
@bin/coverage report --show-missing
@bin/coverage html --directory ./coverage
production: bin/buildout production.cfg setup.py enginescfg
production: bin/buildout production.cfg setup.py
bin/buildout -c production.cfg $(options)
@echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`"
@echo "* Hint 1: on production, disable debug mode and change secret_key"
@echo "* Hint 2: searx will be executed at server startup by crontab"
@echo "* Hint 3: to run immediatley, execute 'bin/supervisord'"
minimal: bin/buildout minimal.cfg setup.py enginescfg
minimal: bin/buildout minimal.cfg setup.py
bin/buildout -c minimal.cfg $(options)
clean:
@rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs \
searx.egg-info lib include .coverage coverage
.PHONY: all tests enginescfg robot flake8 coverage production minimal clean
.PHONY: all tests robot flake8 coverage production minimal clean

2
buildout.cfg
View File

@ -16,8 +16,6 @@ recipe = zc.recipe.egg:script
eggs = ${buildout:eggs}
interpreter = py
dependent-scripts = true
entry-points =
searx-run=searx.webapp:run
[robot]

2
minimal.cfg
View File

@ -13,5 +13,3 @@ parts +=
recipe = zc.recipe.egg:script
eggs = ${buildout:eggs}
interpreter = py
entry-points =
searx-run=searx.webapp:run

2
production.cfg
View File

@ -15,8 +15,6 @@ parts +=
recipe = zc.recipe.egg:script
eggs = ${buildout:eggs}
interpreter = py
entry-points =
searx-run=searx.webapp:run
[supervisor]

6
searx/__init__.py
View File

@ -1,5 +1,5 @@
from os import environ
from os.path import realpath, dirname, join
from os.path import realpath, dirname, join, abspath
try:
from yaml import load
except:
@ -7,8 +7,7 @@ except:
stderr.write('[E] install pyyaml\n')
exit(2)
searx_dir = realpath(dirname(realpath(__file__))+'/../')
searx_dir = abspath(dirname(__file__))
engine_dir = dirname(realpath(__file__))
if 'SEARX_SETTINGS_PATH' in environ:
@ -19,4 +18,3 @@ else:
with open(settings_path) as settings_yaml:
settings = load(settings_yaml)

86
searx/engines/__init__.py
View File

@ -35,6 +35,7 @@ engines = {}
categories = {'general': []}
def load_module(filename):
modname = splitext(filename)[0]
if modname in sys.modules:
@ -58,38 +59,50 @@ for engine_data in settings['engines']:
if engine_data['categories'] == 'none':
engine.categories = []
else:
engine.categories = map(str.strip, engine_data['categories'].split(','))
engine.categories = map(
str.strip, engine_data['categories'].split(','))
continue
setattr(engine, param_name, engine_data[param_name])
for engine_attr in dir(engine):
if engine_attr.startswith('_'):
continue
if getattr(engine, engine_attr) == None:
print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr)
print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr) # noqa
sys.exit(1)
engines[engine.name] = engine
engine.stats = {'result_count': 0, 'search_count': 0, 'page_load_time': 0, 'score_count': 0, 'errors': 0}
engine.stats = {
'result_count': 0,
'search_count': 0,
'page_load_time': 0,
'score_count': 0,
'errors': 0
}
if hasattr(engine, 'categories'):
for category_name in engine.categories:
categories.setdefault(category_name, []).append(engine)
else:
categories['general'].append(engine)
def default_request_params():
return {'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
return {
'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
def make_callback(engine_name, results, suggestions, callback, params):
# creating a callback wrapper for the search engine results
def process_callback(response, **kwargs):
cb_res = []
response.search_params = params
engines[engine_name].stats['page_load_time'] += (datetime.now() - params['started']).total_seconds()
engines[engine_name].stats['page_load_time'] += \
(datetime.now() - params['started']).total_seconds()
try:
search_results = callback(response)
except Exception, e:
engines[engine_name].stats['errors'] += 1
results[engine_name] = cb_res
print '[E] Error with engine "{0}":\n\t{1}'.format(engine_name, str(e))
print '[E] Error with engine "{0}":\n\t{1}'.format(
engine_name, str(e))
return
for result in search_results:
result['engine'] = engine_name
@ -101,8 +114,10 @@ def make_callback(engine_name, results, suggestions, callback, params):
results[engine_name] = cb_res
return process_callback
def score_results(results):
flat_res = filter(None, chain.from_iterable(izip_longest(*results.values())))
flat_res = filter(
None, chain.from_iterable(izip_longest(*results.values())))
flat_len = len(flat_res)
engines_len = len(results)
results = []
@ -116,8 +131,8 @@ def score_results(results):
score = int((flat_len - i) / engines_len) * weight + 1
duplicated = False
for new_res in results:
p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path
p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path
p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
if res['parsed_url'].netloc == new_res['parsed_url'].netloc and\
p1 == p2 and\
res['parsed_url'].query == new_res['parsed_url'].query and\
@ -125,7 +140,7 @@ def score_results(results):
duplicated = new_res
break
if duplicated:
if len(res.get('content', '')) > len(duplicated.get('content', '')):
if len(res.get('content', '')) > len(duplicated.get('content', '')): # noqa
duplicated['content'] = res['content']
duplicated['score'] += score
duplicated['engines'].append(res['engine'])
@ -139,6 +154,7 @@ def score_results(results):
results.append(res)
return sorted(results, key=itemgetter('score'), reverse=True)
def search(query, request, selected_engines):
global engines, categories, number_of_searches
requests = []
@ -160,12 +176,19 @@ def search(query, request, selected_engines):
request_params['started'] = datetime.now()
request_params = engine.request(query, request_params)
callback = make_callback(selected_engine['name'], results, suggestions, engine.response, request_params)
callback = make_callback(
selected_engine['name'],
results,
suggestions,
engine.response,
request_params
)
request_args = dict(headers = request_params['headers']
,hooks = dict(response=callback)
,cookies = request_params['cookies']
,timeout = settings['server']['request_timeout']
request_args = dict(
headers=request_params['headers'],
hooks=dict(response=callback),
cookies=request_params['cookies'],
timeout=settings['server']['request_timeout']
)
if request_params['method'] == 'GET':
@ -192,6 +215,7 @@ def search(query, request, selected_engines):
return results, suggestions
def get_engines_stats():
# TODO refactor
pageloads = []
@ -200,14 +224,15 @@ def get_engines_stats():
errors = []
scores_per_result = []
max_pageload = max_results = max_score = max_errors = max_score_per_result = 0
max_pageload = max_results = max_score = max_errors = max_score_per_result = 0 # noqa
for engine in engines.values():
if engine.stats['search_count'] == 0:
continue
results_num = engine.stats['result_count']/float(engine.stats['search_count'])
load_times = engine.stats['page_load_time']/float(engine.stats['search_count'])
results_num = \
engine.stats['result_count'] / float(engine.stats['search_count'])
load_times = engine.stats['page_load_time'] / float(engine.stats['search_count']) # noqa
if results_num:
score = engine.stats['score_count'] / float(engine.stats['search_count'])
score = engine.stats['score_count'] / float(engine.stats['search_count']) # noqa
score_per_result = score / results_num
else:
score = score_per_result = 0.0
@ -220,7 +245,10 @@ def get_engines_stats():
results.append({'avg': results_num, 'name': engine.name})
scores.append({'avg': score, 'name': engine.name})
errors.append({'avg': engine.stats['errors'], 'name': engine.name})
scores_per_result.append({'avg': score_per_result, 'name': engine.name})
scores_per_result.append({
'avg': score_per_result,
'name': engine.name
})
for engine in pageloads:
engine['percentage'] = int(engine['avg'] / max_pageload * 100)
@ -240,10 +268,16 @@ def get_engines_stats():
else:
engine['percentage'] = 0
return [('Page loads (sec)', sorted(pageloads, key=itemgetter('avg')))
,('Number of results', sorted(results, key=itemgetter('avg'), reverse=True))
,('Scores', sorted(scores, key=itemgetter('avg'), reverse=True))
,('Scores per result', sorted(scores_per_result, key=itemgetter('avg'), reverse=True))
,('Errors', sorted(errors, key=itemgetter('avg'), reverse=True))
return [
('Page loads (sec)', sorted(pageloads, key=itemgetter('avg'))),
(
'Number of results',
sorted(results, key=itemgetter('avg'), reverse=True)
),
('Scores', sorted(scores, key=itemgetter('avg'), reverse=True)),
(
'Scores per result',
sorted(scores_per_result, key=itemgetter('avg'), reverse=True)
),
('Errors', sorted(errors, key=itemgetter('avg'), reverse=True)),
]

3
searx/engines/bing.py
View File

@ -8,7 +8,8 @@ locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx
def request(query, params):
search_path = search_string.format(query=urlencode({'q': query, 'setmkt': locale}))
search_path = search_string.format(
query=urlencode({'q': query, 'setmkt': locale}))
#if params['category'] == 'images':
# params['url'] = base_url + 'images/' + search_path
params['url'] = base_url + search_path

19
searx/engines/currency_convert.py
View File

@ -7,6 +7,7 @@ weight = 100
parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I)
def request(query, params):
m = parser_re.match(query)
if not m:
@ -38,19 +39,21 @@ def response(resp):
except:
return results
title = '{0} {1} in {2} is {3}'.format(resp.search_params['ammount']
,resp.search_params['from']
,resp.search_params['to']
,resp.search_params['ammount']*conversion_rate
title = '{0} {1} in {2} is {3}'.format(
resp.search_params['ammount'],
resp.search_params['from'],
resp.search_params['to'],
resp.search_params['ammount'] * conversion_rate
)
content = '1 {0} is {1} {2}'.format(resp.search_params['from'], conversion_rate, resp.search_params['to'])
now_date = datetime.now().strftime('%Y%m%d')
url = 'http://finance.yahoo.com/currency/converter-results/{0}/{1}-{2}-to-{3}.html'
url = url.format(now_date
,resp.search_params['ammount']
,resp.search_params['from'].lower()
,resp.search_params['to'].lower()
url = url.format(
now_date,
resp.search_params['ammount'],
resp.search_params['from'].lower(),
resp.search_params['to'].lower()
)
results.append({'title': title, 'content': content, 'url': url})

5
searx/engines/dailymotion.py
View File

@ -8,9 +8,11 @@ locale = 'en_US'
# see http://www.dailymotion.com/doc/api/obj-video.html
search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}'
def request(query, params):
global search_url
params['url'] = search_url.format(query=urlencode({'search': query, 'localization': locale }))
params['url'] = search_url.format(
query=urlencode({'search': query, 'localization': locale}))
return params
@ -32,6 +34,7 @@ def response(resp):
results.append({'url': url, 'title': title, 'content': content})
return results
def text_content_from_html(html_string):
desc_html = html.fragment_fromstring(html_string, create_parent=True)
return desc_html.text_content()

0
settings.yml → searx/settings.yml
View File

16
searx/settings_robot.py
View File

@ -1,16 +0,0 @@
port = 11111
secret_key = "ultrasecretkey" # change this!
debug = False
request_timeout = 5.0 # seconds
weights = {} # 'search_engine_name': float(weight) | default is 1.0
blacklist = [] # search engine blacklist
categories = {} # custom search engine categories
base_url = None # "https://your.domain.tld/" or None (to use request parameters)

107
searx/settings_robot.yml Normal file
View File

@ -0,0 +1,107 @@
server:
port : 11111
secret_key : "ultrasecretkey" # change this!
debug : False
request_timeout : 3.0 # seconds
base_url: False
engines:
- name : wikipedia
engine : mediawiki
url : https://en.wikipedia.org/
number_of_results : 1
- name : bing
engine : bing
locale : en-US
- name : currency
engine : currency_convert
categories : general
- name : deviantart
engine : deviantart
categories : images
- name : ddg definitions
engine : duckduckgo_definitions
- name : duckduckgo
engine : duckduckgo
locale : en-us
- name : filecrop
engine : filecrop
categories : files
- name : flickr
engine : flickr
categories : images
- name : github
engine : github
categories : it
- name : google
engine : json_engine
search_url : https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
categories : general
url_query : /responseData/results/unescapedUrl
content_query : /responseData/results/content
title_query : /responseData/results/titleNoFormatting
- name : google images
engine : google_images
categories : images
- name : piratebay
engine : piratebay
categories : videos, music, files
- name : soundcloud
engine : soundcloud
categories : music
- name : stackoverflow
engine : stackoverflow
categories : it
- name : startpage
engine : startpage
- name : twitter
engine : twitter
categories : social media
- name : urbandictionary
engine : xpath
search_url : http://www.urbandictionary.com/define.php?term={query}
url_xpath : //div[@class="word"]//a/@href
title_xpath : //div[@class="word"]//a
content_xpath : //div[@class="definition"]
- name : yahoo
engine : xpath
search_url : http://search.yahoo.com/search?p={query}
results_xpath : //div[@class="res"]
url_xpath : .//h3/a/@href
title_xpath : .//h3/a
content_xpath : .//div[@class="abstr"]
suggestion_xpath : //div[@id="satat"]//a
- name : youtube
engine : youtube
categories : videos
- name : dailymotion
engine : dailymotion
locale : en_US
categories : videos
- name : vimeo
engine : vimeo
categories : videos
results_xpath : //div[@id="browse_content"]/ol/li
url_xpath : ./a/@href
title_xpath : ./a/div[@class="data"]/p[@class="title"]/text()
content_xpath : ./a/img/@src

19
searx/testing.py
View File

@ -7,10 +7,10 @@ from unittest2 import TestCase
import os
import subprocess
import sys
class SearxTestLayer:
"""Base layer for non-robot tests."""
__name__ = u'SearxTestLayer'
@ -36,24 +36,37 @@ class SearxRobotLayer(Layer):
def setUp(self):
os.setpgrp() # create new process group, become its leader
# get program paths
webapp = os.path.join(
os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
'webapp.py'
)
exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py')
# set robot settings path
os.environ['SEARX_SETTINGS_PATH'] = os.path.abspath(
os.path.dirname(__file__) + '/settings_robot.yml')
# run the server
self.server = subprocess.Popen(
[exe, webapp, 'settings_robot'],
[exe, webapp],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
def tearDown(self):
# TERM all processes in my group
# send TERM signal to all processes in my group, to stop subprocesses
os.killpg(os.getpgid(self.server.pid), 15)
# remove previously set environment variable
del os.environ['SEARX_SETTINGS_PATH']
SEARXROBOTLAYER = SearxRobotLayer()
class SearxTestCase(TestCase):
"""Base test case for non-robot tests."""
layer = SearxTestLayer

4
searx/utils.py
View File

@ -5,10 +5,12 @@ import codecs
import cStringIO
import re
def gen_useragent():
# TODO
return "Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"
def highlight_content(content, query):
if not content:
@ -34,6 +36,7 @@ def highlight_content(content, query):
return content
class HTMLTextExtractor(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
@ -54,6 +57,7 @@ class HTMLTextExtractor(HTMLParser):
def get_text(self):
return u''.join(self.result)
def html_to_text(html):
s = HTMLTextExtractor()
s.feed(html)

57
searx/webapp.py
View File

@ -17,13 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
import os
import sys
if __name__ == "__main__":
sys.path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../'))
from searx import settings
from flask import Flask, request, render_template, url_for, Response, make_response, redirect
from searx.engines import search, categories, engines, get_engines_stats
import json
@ -33,11 +27,17 @@ from flask import send_from_directory
from searx.utils import highlight_content, html_to_text
import os
app = Flask(
__name__,
static_folder=os.path.join(os.path.dirname(__file__), 'static'),
template_folder=os.path.join(os.path.dirname(__file__), 'templates')
)
app = Flask(__name__)
app.secret_key = settings['server']['secret_key']
#TODO configurable via settings.yml
favicons = ['wikipedia', 'youtube', 'vimeo', 'soundcloud',
'twitter', 'stackoverflow', 'github']
@ -81,6 +81,7 @@ def render(template_name, **kwargs):
kwargs['selected_categories'] = ['general']
return render_template(template_name, **kwargs)
def parse_query(query):
query_engines = []
query_parts = query.split()
@ -159,22 +160,23 @@ def index():
response.headers.add('Content-Disposition', 'attachment;Filename=searx_-_{0}.csv'.format('_'.join(query.split())))
return response
elif request_data.get('format') == 'rss':
response_rss = render('opensearch_response_rss.xml'
,results=results
,q=request_data['q']
,number_of_results=len(results)
,base_url=get_base_url()
response_rss = render(
'opensearch_response_rss.xml',
results=results,
q=request_data['q'],
number_of_results=len(results),
base_url=get_base_url()
)
return Response(response_rss, mimetype='text/xml')
return render('results.html'
,results=results
,q=request_data['q']
,selected_categories=selected_categories
,number_of_results=len(results)+len(featured_results)
,featured_results=featured_results
,suggestions=suggestions
return render(
'results.html',
results=results,
q=request_data['q'],
selected_categories=selected_categories,
number_of_results=len(results) + len(featured_results),
featured_results=featured_results,
suggestions=suggestions
)
@ -203,7 +205,10 @@ def preferences():
if selected_categories:
resp = make_response(redirect('/'))
# cookie max age: 4 weeks
resp.set_cookie('categories', ','.join(selected_categories), max_age=60*60*24*7*4)
resp.set_cookie(
'categories', ','.join(selected_categories),
max_age=60 * 60 * 24 * 7 * 4
)
return resp
return render('preferences.html')
@ -238,6 +243,7 @@ def opensearch():
mimetype="application/xml")
return resp
@app.route('/favicon.ico')
def favicon():
return send_from_directory(os.path.join(app.root_path, 'static/img'),
@ -248,9 +254,10 @@ def run():
from gevent import monkey
monkey.patch_all()
app.run(debug = settings['server']['debug']
,use_debugger = settings['server']['debug']
,port = settings['server']['port']
app.run(
debug=settings['server']['debug'],
use_debugger=settings['server']['debug'],
port=settings['server']['port']
)

15
setup.py
View File

@ -49,4 +49,19 @@ setup(
'zope.testrunner',
]
},
entry_points={
'console_scripts': [
'searx-run = searx.webapp:run'
]
},
package_data={
'searx': [
'settings.yml',
'../README.md',
'static/*/*',
'templates/*.html',
'templates/result_templates/*.html',
],
},
)