kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-22 12:10:11 +01:00
Code

[mod] search refactor

Browse Source
This commit is contained in:
asciimoo 2013-10-15 18:19:06 +02:00
parent 2440e74b1e
commit d5ec0f43e4
2 changed files with 40 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
searx/engines/__init__.py
View File

@ -2,6 +2,7 @@
from os.path import realpath, dirname, splitext, join from os.path import realpath, dirname, splitext, join
from os import listdir from os import listdir
from imp import load_source from imp import load_source
import grequests
engine_dir = dirname(realpath(__file__)) engine_dir = dirname(realpath(__file__))
@ -12,4 +13,40 @@ for filename in listdir(engine_dir):
if filename.startswith('_') or not filename.endswith('.py'): if filename.startswith('_') or not filename.endswith('.py'):
continue continue
filepath = join(engine_dir, filename) filepath = join(engine_dir, filename)
engines.append(load_source(modname, filepath)) engine = load_source(modname, filepath)
if not hasattr(engine, 'request') or not hasattr(engine, 'response'):
continue
engines.append(engine)
def default_request_params():
return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}
def make_callback(results, callback):
def process_callback(response, **kwargs):
results.extend(callback(response))
return process_callback
def search(query, request):
global engines
requests = []
results = []
user_agent = request.headers.get('User-Agent', '')
for engine in engines:
headers = default_request_params()
headers['User-Agent'] = user_agent
request_params = engine.request(query, headers)
callback = make_callback(results, engine.response)
if request_params['method'] == 'GET':
req = grequests.get(request_params['url']
,headers=headers
,hooks=dict(response=callback)
)
else:
req = grequests.post(request_params['url']
,data=request_params['data']
,headers=headers
,hooks=dict(response=callback)
)
requests.append(req)
grequests.map(requests)
return results

35
searx/webapp.py
View File

@ -25,8 +25,7 @@ if __name__ == "__main__":
from flask import Flask, request, flash, render_template from flask import Flask, request, flash, render_template
import ConfigParser import ConfigParser
from os import getenv from os import getenv
from searx.engines import engines from searx.engines import search
import grequests
cfg = ConfigParser.SafeConfigParser() cfg = ConfigParser.SafeConfigParser()
cfg.read('/etc/searx.conf') cfg.read('/etc/searx.conf')
@ -38,14 +37,6 @@ cfg.read('searx.conf')
app = Flask(__name__) app = Flask(__name__)
app.secret_key = cfg.get('app', 'secret_key') app.secret_key = cfg.get('app', 'secret_key')
def default_request_params():
return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}
def make_callback(results, callback):
def process_callback(response, **kwargs):
results.extend(callback(response))
return process_callback
@app.route('/', methods=['GET', 'POST']) @app.route('/', methods=['GET', 'POST'])
def index(): def index():
if request.method=='POST': if request.method=='POST':
@ -53,30 +44,8 @@ def index():
flash('Wrong post data') flash('Wrong post data')
return render_template('index.html') return render_template('index.html')
query = request.form['q'] query = request.form['q']
requests = [] results = search(query, request)
results = []
user_agent = request.headers.get('User-Agent', '')
for engine in engines:
headers = default_request_params()
headers['User-Agent'] = user_agent
request_params = engine.request(query, headers)
callback = make_callback(results, engine.response)
if request_params['method'] == 'GET':
req = grequests.get(request_params['url']
,headers=headers
,hooks=dict(response=callback)
)
else:
req = grequests.post(request_params['url']
,data=request_params['data']
,headers=headers
,hooks=dict(response=callback)
)
requests.append(req)
grequests.map(requests)
return render_template('results.html', results=results, q=query) return render_template('results.html', results=results, q=query)
return render_template('index.html') return render_template('index.html')
if __name__ == "__main__": if __name__ == "__main__":