From 29a526ff0e33030fc9d1416a9ccfeba93e376664 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 15:24:21 +0100 Subject: [PATCH 1/8] [enh] add rss-link which can be detected by browsers --- searx/templates/oscar/base.html | 1 + searx/templates/oscar/results.html | 1 + 2 files changed, 2 insertions(+) diff --git a/searx/templates/oscar/base.html b/searx/templates/oscar/base.html index e46024d17..466756b6f 100644 --- a/searx/templates/oscar/base.html +++ b/searx/templates/oscar/base.html @@ -7,6 +7,7 @@ + {% block meta %}{% endblock %} {% block title %}{% endblock %}searx diff --git a/searx/templates/oscar/results.html b/searx/templates/oscar/results.html index 380947ca0..46c230695 100644 --- a/searx/templates/oscar/results.html +++ b/searx/templates/oscar/results.html @@ -1,5 +1,6 @@ {% extends "oscar/base.html" %} {% block title %}{{ q }} - {% endblock %} +{% block meta %}{% endblock %} {% block content %}
From 4e2dae30f013d8ef9fd6d22e3f01e38f13f13c11 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 16:42:57 +0100 Subject: [PATCH 2/8] [enh] add autocompletion for searx-specific strings --- searx/autocomplete.py | 79 +++++++++++++++++++++++++++++++++++++++++++ searx/query.py | 2 +- searx/webapp.py | 7 ++-- 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 545bd69e7..ad5099347 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -20,6 +20,85 @@ from lxml import etree from requests import get from json import loads from urllib import urlencode +from searx.languages import language_codes +from searx.engines import ( + categories, engines, engine_shortcuts +) + + +def searx_bang(full_query): + '''check if the searchQuery contain a bang, and create fitting autocompleter results''' + # check if there is a query which can be parsed + if len(full_query.getSearchQuery()) == 0: + return [] + + results = [] + + # check if current query stats with !bang + if full_query.getSearchQuery()[0] == '!': + if len(full_query.getSearchQuery()) == 1: + # show some example queries + # TODO, check if engine is not avaliable + results.append("!images") + results.append("!wikipedia") + results.append("!osm") + else: + engine_query = full_query.getSearchQuery()[1:] + + # check if query starts with categorie name + for categorie in categories: + if categorie.startswith(engine_query): + results.append('!{categorie}'.format(categorie=categorie)) + + # check if query starts with engine name + for engine in engines: + if engine.startswith(engine_query): + results.append('!{engine}'.format(engine=engine.replace(' ', '_'))) + + # check if query starts with engine shortcut + for engine_shortcut in engine_shortcuts: + if engine_shortcut.startswith(engine_query): + results.append('!{engine_shortcut}'.format(engine_shortcut=engine_shortcut)) + + # check if current query stats with :bang + elif full_query.getSearchQuery()[0] == ':': + if len(full_query.getSearchQuery()) == 1: + # show some example queries + results.append(":en") + results.append(":en_us") + results.append(":english") + results.append(":united_kingdom") + else: + engine_query = full_query.getSearchQuery()[1:] + + for lc in language_codes: + lang_id, lang_name, country = map(str.lower, lc) + + # check if query starts with language-id + if lang_id.startswith(engine_query): + if len(engine_query) <= 2: + results.append(':{lang_id}'.format(lang_id=lang_id.split('_')[0])) + else: + results.append(':{lang_id}'.format(lang_id=lang_id)) + + # check if query starts with language name + if lang_name.startswith(engine_query): + results.append(':{lang_name}'.format(lang_name=lang_name)) + + # check if query starts with country + if country.startswith(engine_query): + results.append(':{country}'.format(country=country.replace(' ', '_'))) + + # remove duplicates + result_set = set(results) + + # remove results which are already contained in the query + for query_part in full_query.query_parts: + if query_part in result_set: + result_set.remove(query_part) + + # convert result_set back to list + return list(result_set) def dbpedia(query): diff --git a/searx/query.py b/searx/query.py index 9f711e982..cd2b3f423 100644 --- a/searx/query.py +++ b/searx/query.py @@ -77,7 +77,7 @@ class Query(object): if lang == lang_id\ or lang_id.startswith(lang)\ or lang == lang_name\ - or lang == country: + or lang.replace('_', ' ') == country: parse_next = True self.languages.append(lang) break diff --git a/searx/webapp.py b/searx/webapp.py index 93af05083..3dcba2968 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -46,7 +46,7 @@ from searx.languages import language_codes from searx.https_rewrite import https_url_rewrite from searx.search import Search from searx.query import Query -from searx.autocomplete import backends as autocomplete_backends +from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx import logger @@ -352,8 +352,11 @@ def autocompleter(): if not completer: return + # parse searx specific autocompleter results like !bang + raw_results = searx_bang(query) + # run autocompletion - raw_results = completer(query.getSearchQuery()) + raw_results.extend(completer(query.getSearchQuery())) # parse results (write :language and !engine back to result string) results = [] From 830f70a6bc2a6e0b7a78d0a13277b28f03c18504 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 19:40:27 +0100 Subject: [PATCH 3/8] [enh] improve torrent results --- searx/engines/kickass.py | 31 +++++++++++++++++++ .../oscar/result_templates/torrent.html | 16 ++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py index 16e9d6de6..39b50b0ef 100644 --- a/searx/engines/kickass.py +++ b/searx/engines/kickass.py @@ -24,6 +24,7 @@ search_url = url + 'search/{search_term}/{pageno}/' # specific xpath variables magnet_xpath = './/a[@title="Torrent magnet link"]' +torrent_xpath = './/a[@title="Download torrent file"]' content_xpath = './/span[@class="font11px lightgrey block"]' @@ -60,6 +61,9 @@ def response(resp): method="text")) seed = result.xpath('.//td[contains(@class, "green")]/text()')[0] leech = result.xpath('.//td[contains(@class, "red")]/text()')[0] + filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0] + filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0] + files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0] # convert seed to int if possible if seed.isdigit(): @@ -73,7 +77,31 @@ def response(resp): else: leech = 0 + # convert filesize to byte if possible + try: + filesize = float(filesize) + + # convert filesize to byte + if filesize_multiplier == 'TB': + filesize = int(filesize * 1024*1024*1024*1024) + elif filesize_multiplier == 'GB': + filesize = int(filesize * 1024*1024*1024) + elif filesize_multiplier == 'MB': + filesize = int(filesize * 1024*1024) + elif filesize_multiplier == 'kb': + filesize = int(filesize * 1024) + except: + filesize = None + + # convert files to int if possible + if files.isdigit(): + files = int(files) + else: + files = None + magnetlink = result.xpath(magnet_xpath)[0].attrib['href'] + + torrentfile = result.xpath(torrent_xpath)[0].attrib['href'] # append result results.append({'url': href, @@ -81,7 +109,10 @@ def response(resp): 'content': content, 'seed': seed, 'leech': leech, + 'filesize': filesize, + 'files': files, 'magnetlink': magnetlink, + 'torrentfile':torrentfile, 'template': 'torrent.html'}) # return results sorted by seeder diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html index bfb6cdbe6..f62a7a85b 100644 --- a/searx/templates/oscar/result_templates/torrent.html +++ b/searx/templates/oscar/result_templates/torrent.html @@ -5,9 +5,19 @@ {% if result.publishedDate %}{% endif %} {{ icon('link') }} {{ _('cached') }} -

{{ icon('transfer') }} {{ _('Seeder') }} {{ result.seed }}, {{ _('Leecher') }} {{ result.leech }} -
-{{ icon('magnet') }} magnet link

+

{{ icon('transfer') }} {{ _('Seeder') }} {{ result.seed }} • {{ _('Leecher') }} {{ result.leech }} +{% if result.filesize %}
{{ icon('floppy-disk') }} {{ _('Filesize') }} + + {% if result.filesize < 1024 %}{{ result.filesize }} Byte + {% elif result.filesize < 1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024) }} kb + {% elif result.filesize < 1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024) }} MB + {% elif result.filesize < 1024*1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024) }} GB{% endif %} + {% endif %} +{% if result.files %}
{{ icon('file') }} {{ _('Number of Files') }} {{ result.files }}{% endif %}

+

+ {{ icon('magnet') }} magnet link + {% if result.torrentfile %}
{{ icon('download-alt') }} torrent file{% endif %} +

{% if result.content %}

{{ result.content|safe }}

{% endif %} From 08f820d74c7de4309fe8227da5c10e4179cef54f Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 19:42:41 +0100 Subject: [PATCH 4/8] [enh] make some torrent strings translatable --- searx/templates/oscar/result_templates/torrent.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html index f62a7a85b..5268869ca 100644 --- a/searx/templates/oscar/result_templates/torrent.html +++ b/searx/templates/oscar/result_templates/torrent.html @@ -15,8 +15,8 @@ {% endif %} {% if result.files %}
{{ icon('file') }} {{ _('Number of Files') }} {{ result.files }}{% endif %}

- {{ icon('magnet') }} magnet link - {% if result.torrentfile %}
{{ icon('download-alt') }} torrent file{% endif %} + {{ icon('magnet') }} {{ _('magnet link') }} + {% if result.torrentfile %}
{{ icon('download-alt') }} {{ _('torrent file') }}{% endif %}

{% if result.content %}

{{ result.content|safe }}

{% endif %} From eb6f3348c43f3456fa40716589967ec8a5ec53e2 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 19:44:08 +0100 Subject: [PATCH 5/8] [enh] display filesize greater 1TB --- searx/templates/oscar/result_templates/torrent.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html index 5268869ca..9258a0cb7 100644 --- a/searx/templates/oscar/result_templates/torrent.html +++ b/searx/templates/oscar/result_templates/torrent.html @@ -11,7 +11,8 @@ {% if result.filesize < 1024 %}{{ result.filesize }} Byte {% elif result.filesize < 1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024) }} kb {% elif result.filesize < 1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024) }} MB - {% elif result.filesize < 1024*1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024) }} GB{% endif %} + {% elif result.filesize < 1024*1024*1024*1024 %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024) }} GB + {% else %}{{ '{0:0.2f}'.format(result.filesize/1024/1024/1024/1024) }} TB{% endif %} {% endif %} {% if result.files %}
{{ icon('file') }} {{ _('Number of Files') }} {{ result.files }}{% endif %}

From c19b0899a415fa3be7c1f30258c218868ef0b671 Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 19:55:21 +0100 Subject: [PATCH 6/8] [fix] little autocompleter fix --- searx/autocomplete.py | 4 ++-- searx/webapp.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/searx/autocomplete.py b/searx/autocomplete.py index ad5099347..7ebebf1de 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -52,7 +52,7 @@ def searx_bang(full_query): # check if query starts with engine name for engine in engines: - if engine.startswith(engine_query): + if engine.startswith(engine_query.replace('_', ' ')): results.append('!{engine}'.format(engine=engine.replace(' ', '_'))) # check if query starts with engine shortcut @@ -86,7 +86,7 @@ def searx_bang(full_query): results.append(':{lang_name}'.format(lang_name=lang_name)) # check if query starts with country - if country.startswith(engine_query): + if country.startswith(engine_query.replace('_', ' ')): results.append(':{country}'.format(country=country.replace(' ', '_'))) # remove duplicates diff --git a/searx/webapp.py b/searx/webapp.py index 3dcba2968..bd79e0035 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -355,8 +355,10 @@ def autocompleter(): # parse searx specific autocompleter results like !bang raw_results = searx_bang(query) - # run autocompletion - raw_results.extend(completer(query.getSearchQuery())) + # normal autocompletion results only appear if max 3. searx results returned + if len(raw_results) <= 3: + # run autocompletion + raw_results.extend(completer(query.getSearchQuery())) # parse results (write :language and !engine back to result string) results = [] From 811515101b750717f6b07fad1a9b75041f91c90d Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 20:01:36 +0100 Subject: [PATCH 7/8] [fix] pep8 --- searx/engines/kickass.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py index 39b50b0ef..a4d270673 100644 --- a/searx/engines/kickass.py +++ b/searx/engines/kickass.py @@ -80,16 +80,16 @@ def response(resp): # convert filesize to byte if possible try: filesize = float(filesize) - + # convert filesize to byte if filesize_multiplier == 'TB': - filesize = int(filesize * 1024*1024*1024*1024) + filesize = int(filesize * 1024 * 1024 * 1024 * 1024) elif filesize_multiplier == 'GB': - filesize = int(filesize * 1024*1024*1024) + filesize = int(filesize * 1024 * 1024 * 1024) elif filesize_multiplier == 'MB': - filesize = int(filesize * 1024*1024) + filesize = int(filesize * 1024 * 1024) elif filesize_multiplier == 'kb': - filesize = int(filesize * 1024) + filesize = int(filesize * 1024) except: filesize = None @@ -100,7 +100,7 @@ def response(resp): files = None magnetlink = result.xpath(magnet_xpath)[0].attrib['href'] - + torrentfile = result.xpath(torrent_xpath)[0].attrib['href'] # append result @@ -112,7 +112,7 @@ def response(resp): 'filesize': filesize, 'files': files, 'magnetlink': magnetlink, - 'torrentfile':torrentfile, + 'torrentfile': torrentfile, 'template': 'torrent.html'}) # return results sorted by seeder From 9109686c4e7208e0c228afcc8dfe5e20e24d598d Mon Sep 17 00:00:00 2001 From: Thomas Pointhuber Date: Sat, 10 Jan 2015 20:26:17 +0100 Subject: [PATCH 8/8] [enh] using less html code to disable autoescaping for embeded results --- searx/templates/oscar/result_templates/default.html | 4 +--- searx/templates/oscar/result_templates/videos.html | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/searx/templates/oscar/result_templates/default.html b/searx/templates/oscar/result_templates/default.html index 25506b2f5..b2430ed6a 100644 --- a/searx/templates/oscar/result_templates/default.html +++ b/searx/templates/oscar/result_templates/default.html @@ -11,9 +11,7 @@ {% if result.embedded %}

-{% autoescape false %} - {{ result.embedded }} -{% endautoescape %} + {{ result.embedded|safe }}
{% endif %} diff --git a/searx/templates/oscar/result_templates/videos.html b/searx/templates/oscar/result_templates/videos.html index 32000e662..c3d02c14d 100644 --- a/searx/templates/oscar/result_templates/videos.html +++ b/searx/templates/oscar/result_templates/videos.html @@ -11,9 +11,7 @@ {% if result.embedded %}
-{% autoescape false %} - {{ result.embedded }} -{% endautoescape %} + {{ result.embedded|safe }}
{% endif %}