kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-19 02:40:11 +01:00
Code

Merge pull request #621 from stepshal/anomalous-backslash-in-string

Browse Source 
Fix anomalous backslash in string
This commit is contained in:
Adam Tauber 2016-07-18 22:27:17 +02:00 committed by GitHub
commit aa09f963eb
21 changed files with 47 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
searx/engines/currency_convert.py
View File

@ -9,7 +9,7 @@ categories = []
url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X' url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'
weight = 100 weight = 100
parser_re = re.compile(u'.*?(\d+(?:\.\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I) # noqa parser_re = re.compile(u'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I) # noqa
db = 1 db = 1

2
searx/engines/deviantart.py
View File

@ -47,7 +47,7 @@ def response(resp):
dom = html.fromstring(resp.text) dom = html.fromstring(resp.text)
regex = re.compile('\/200H\/') regex = re.compile(r'\/200H\/')
# parse results # parse results
for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'): for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):

4
searx/engines/google.py
View File

@ -300,9 +300,9 @@ def parse_map_detail(parsed_url, result, google_hostname):
results = [] results = []
# try to parse the geoloc # try to parse the geoloc
m = re.search('@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path) m = re.search(r'@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path)
if m is None: if m is None:
m = re.search('ll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query) m = re.search(r'll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query)
if m is not None: if m is not None:
# geoloc found (ignored) # geoloc found (ignored)

10
searx/engines/startpage.py
View File

@ -68,15 +68,15 @@ def response(resp):
url = link.attrib.get('href') url = link.attrib.get('href')
# block google-ad url's # block google-ad url's
if re.match("^http(s|)://(www\.)?google\.[a-z]+/aclk.*$", url): if re.match(r"^http(s|)://(www\.)?google\.[a-z]+/aclk.*$", url):
continue continue
# block startpage search url's # block startpage search url's
if re.match("^http(s|)://(www\.)?startpage\.com/do/search\?.*$", url): if re.match(r"^http(s|)://(www\.)?startpage\.com/do/search\?.*$", url):
continue continue
# block ixquick search url's # block ixquick search url's
if re.match("^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url): if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):
continue continue
title = escape(extract_text(link)) title = escape(extract_text(link))
@ -89,7 +89,7 @@ def response(resp):
published_date = None published_date = None
# check if search result starts with something like: "2 Sep 2014 ... " # check if search result starts with something like: "2 Sep 2014 ... "
if re.match("^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]{2} [0-9]{4} \.\.\. ", content): if re.match(r"^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]{2} [0-9]{4} \.\.\. ", content):
date_pos = content.find('...') + 4 date_pos = content.find('...') + 4
date_string = content[0:date_pos - 5] date_string = content[0:date_pos - 5]
published_date = parser.parse(date_string, dayfirst=True) published_date = parser.parse(date_string, dayfirst=True)
@ -98,7 +98,7 @@ def response(resp):
content = content[date_pos:] content = content[date_pos:]
# check if search result starts with something like: "5 days ago ... " # check if search result starts with something like: "5 days ago ... "
elif re.match("^[0-9]+ days? ago \.\.\. ", content): elif re.match(r"^[0-9]+ days? ago \.\.\. ", content):
date_pos = content.find('...') + 4 date_pos = content.find('...') + 4
date_string = content[0:date_pos - 5] date_string = content[0:date_pos - 5]

8
searx/engines/swisscows.py
View File

@ -25,10 +25,10 @@ base_url = 'https://swisscows.ch/'
search_string = '?{query}&page={page}' search_string = '?{query}&page={page}'
# regex # regex
regex_json = re.compile('initialData: {"Request":(.|\n)*},\s*environment') regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment')
regex_json_remove_start = re.compile('^initialData:\s*') regex_json_remove_start = re.compile(r'^initialData:\s*')
regex_json_remove_end = re.compile(',\s*environment$') regex_json_remove_end = re.compile(r',\s*environment$')
regex_img_url_remove_start = re.compile('^https?://i\.swisscows\.ch/\?link=') regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')
# do search-request # do search-request

2
searx/engines/tokyotoshokan.py
View File

@ -48,7 +48,7 @@ def response(resp):
return [] return []
# regular expression for parsing torrent size strings # regular expression for parsing torrent size strings
size_re = re.compile('Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE) size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)
# processing the results, two rows at a time # processing the results, two rows at a time
for i in xrange(0, len(rows), 2): for i in xrange(0, len(rows), 2):

2
searx/engines/www500px.py
View File

@ -41,7 +41,7 @@ def response(resp):
results = [] results = []
dom = html.fromstring(resp.text) dom = html.fromstring(resp.text)
regex = re.compile('3\.jpg.*$') regex = re.compile(r'3\.jpg.*$')
# parse results # parse results
for result in dom.xpath('//div[@class="photo"]'): for result in dom.xpath('//div[@class="photo"]'):

2
searx/engines/yahoo_news.py
View File

@ -55,7 +55,7 @@ def request(query, params):
def sanitize_url(url): def sanitize_url(url):
if ".yahoo.com/" in url: if ".yahoo.com/" in url:
return re.sub(u"\;\_ylt\=.+$", "", url) return re.sub(u"\\;\\_ylt\\=.+$", "", url)
else: else:
return url return url

2
searx/plugins/https_rewrite.py
View File

@ -87,7 +87,7 @@ def load_single_https_ruleset(rules_path):
# convert host-rule to valid regex # convert host-rule to valid regex
host = ruleset.attrib.get('host')\ host = ruleset.attrib.get('host')\
.replace('.', '\.').replace('*', '.*') .replace('.', r'\.').replace('*', '.*')
# append to host list # append to host list
hosts.append(host) hosts.append(host)

2
searx/results.py
View File

@ -5,7 +5,7 @@ from threading import RLock
from urlparse import urlparse, unquote from urlparse import urlparse, unquote
from searx.engines import engines from searx.engines import engines
CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile('[,;:!?\./\\\\ ()-_]', re.M | re.U) CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U) WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)

2
searx/utils.py
View File

@ -63,7 +63,7 @@ def highlight_content(content, query):
regex_parts = [] regex_parts = []
for chunk in query.split(): for chunk in query.split():
if len(chunk) == 1: if len(chunk) == 1:
regex_parts.append(u'\W+{0}\W+'.format(re.escape(chunk))) regex_parts.append(u'\\W+{0}\\W+'.format(re.escape(chunk)))
else: else:
regex_parts.append(u'{0}'.format(re.escape(chunk))) regex_parts.append(u'{0}'.format(re.escape(chunk)))
query_regex = u'({0})'.format('|'.join(regex_parts)) query_regex = u'({0})'.format('|'.join(regex_parts))

2
tests/unit/engines/test_dailymotion.py
View File

@ -62,7 +62,7 @@ class TestDailymotionEngine(SearxTestCase):
self.assertEqual(results[0]['content'], 'Description') self.assertEqual(results[0]['content'], 'Description')
self.assertIn('x2fit7q', results[0]['embedded']) self.assertIn('x2fit7q', results[0]['embedded'])
json = """ json = r"""
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"http:\/\/www.dailymotion.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.dailymotion.com\/artist\/1217","type":"artist"}

4
tests/unit/engines/test_deezer.py
View File

@ -27,7 +27,7 @@ class TestDeezerEngine(SearxTestCase):
response = mock.Mock(text='{"data": []}') response = mock.Mock(text='{"data": []}')
self.assertEqual(deezer.response(response), []) self.assertEqual(deezer.response(response), [])
json = """ json = r"""
{"data":[ {"data":[
{"id":100, "title":"Title of track", {"id":100, "title":"Title of track",
"link":"https:\/\/www.deezer.com\/track\/1094042","duration":232, "link":"https:\/\/www.deezer.com\/track\/1094042","duration":232,
@ -45,7 +45,7 @@ class TestDeezerEngine(SearxTestCase):
self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track') self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')
self.assertTrue('100' in results[0]['embedded']) self.assertTrue('100' in results[0]['embedded'])
json = """ json = r"""
{"data":[ {"data":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"https:\/\/www.deezer.com\/artist\/1217","type":"artist"} "link":"https:\/\/www.deezer.com\/artist\/1217","type":"artist"}

10
tests/unit/engines/test_flickr.py
View File

@ -27,7 +27,7 @@ class TestFlickrEngine(SearxTestCase):
response = mock.Mock(text='{"data": []}') response = mock.Mock(text='{"data": []}')
self.assertEqual(flickr.response(response), []) self.assertEqual(flickr.response(response), [])
json = """ json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [ "photo": [
{ "id": "15751017054", "owner": "66847915@N08", { "id": "15751017054", "owner": "66847915@N08",
@ -55,7 +55,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertTrue('Owner' in results[0]['content']) self.assertTrue('Owner' in results[0]['content'])
self.assertTrue('Description' in results[0]['content']) self.assertTrue('Description' in results[0]['content'])
json = """ json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [ "photo": [
{ "id": "15751017054", "owner": "66847915@N08", { "id": "15751017054", "owner": "66847915@N08",
@ -79,7 +79,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertTrue('Owner' in results[0]['content']) self.assertTrue('Owner' in results[0]['content'])
self.assertTrue('Description' in results[0]['content']) self.assertTrue('Description' in results[0]['content'])
json = """ json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [ "photo": [
{ "id": "15751017054", "owner": "66847915@N08", { "id": "15751017054", "owner": "66847915@N08",
@ -103,7 +103,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertTrue('Owner' in results[0]['content']) self.assertTrue('Owner' in results[0]['content'])
self.assertTrue('Description' in results[0]['content']) self.assertTrue('Description' in results[0]['content'])
json = """ json = r"""
{ "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032", { "photos": { "page": 1, "pages": "41001", "perpage": 100, "total": "4100032",
"photo": [ "photo": [
{ "id": "15751017054", "owner": "66847915@N08", { "id": "15751017054", "owner": "66847915@N08",
@ -130,7 +130,7 @@ class TestFlickrEngine(SearxTestCase):
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
self.assertEqual(len(results), 0) self.assertEqual(len(results), 0)
json = """ json = r"""
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}

2
tests/unit/engines/test_flickr_noapi.py
View File

@ -316,7 +316,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(len(results), 0) self.assertEqual(len(results), 0)
# garbage test # garbage test
json = """ json = r"""
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}

24
tests/unit/engines/test_ina.py
View File

@ -33,23 +33,23 @@ class TestInaEngine(SearxTestCase):
<div class=\\"search-results--list\\"><div class=\\"media\\">\\n\ <div class=\\"search-results--list\\"><div class=\\"media\\">\\n\
\\t\\t\\t\\t<a class=\\"media-left media-video premium xiti_click_action\\" \ \\t\\t\\t\\t<a class=\\"media-left media-video premium xiti_click_action\\" \
data-xiti-params=\\"recherche_v4::resultats_conference_de_presse_du_general_de_gaulle::N\\" \ data-xiti-params=\\"recherche_v4::resultats_conference_de_presse_du_general_de_gaulle::N\\" \
href=\\"\/video\/CAF89035682\/conference-de-presse-du-general-de-gaulle-video.html\\">\\n\ href=\\"\\/video\\/CAF89035682\\/conference-de-presse-du-general-de-gaulle-video.html\\">\\n\
<img src=\\"https:\/\/www.ina.fr\/images_v2\/140x105\/CAF89035682.jpeg\\" \ <img src=\\"https:\\/\\/www.ina.fr\\/images_v2\\/140x105\\/CAF89035682.jpeg\\" \
alt=\\"Conf\\u00e9rence de presse du G\\u00e9n\\u00e9ral de Gaulle \\">\\n\ alt=\\"Conf\\u00e9rence de presse du G\\u00e9n\\u00e9ral de Gaulle \\">\\n\
\\t\\t\\t\\t\\t<\/a>\\n\ \\t\\t\\t\\t\\t<\\/a>\\n\
\\t\\t\\t\\t\\t<div class=\\"media-body\\">\\n\\t\\t\\t\\t\\t\\t<h3 class=\\"h3--title media-heading\\">\\n\ \\t\\t\\t\\t\\t<div class=\\"media-body\\">\\n\\t\\t\\t\\t\\t\\t<h3 class=\\"h3--title media-heading\\">\\n\
\\t\\t\\t\\t\\t\\t\\t<a class=\\"xiti_click_action\\" \ \\t\\t\\t\\t\\t\\t\\t<a class=\\"xiti_click_action\\" \
data-xiti-params=\\"recherche_v4::resultats_conference_de_presse_du_general_de_gaulle::N\\" \ data-xiti-params=\\"recherche_v4::resultats_conference_de_presse_du_general_de_gaulle::N\\" \
href=\\"\/video\/CAF89035682\/conference-de-presse-du-general-de-gaulle-video.html\\">\ href=\\"\\/video\\/CAF89035682\\/conference-de-presse-du-general-de-gaulle-video.html\\">\
Conf\\u00e9rence de presse du G\\u00e9n\\u00e9ral de Gaulle <\/a>\\n\ Conf\\u00e9rence de presse du G\\u00e9n\\u00e9ral de Gaulle <\\/a>\\n\
<\/h3>\\n\ <\\/h3>\\n\
<div class=\\"media-body__info\\">\\n<span class=\\"broadcast\\">27\/11\/1967<\/span>\\n\ <div class=\\"media-body__info\\">\\n<span class=\\"broadcast\\">27\\/11\\/1967<\\/span>\\n\
<span class=\\"views\\">29321 vues<\/span>\\n\ <span class=\\"views\\">29321 vues<\\/span>\\n\
<span class=\\"duration\\">01h 33m 07s<\/span>\\n\ <span class=\\"duration\\">01h 33m 07s<\\/span>\\n\
<\/div>\\n\ <\\/div>\\n\
<p class=\\"media-body__summary\\">VERSION INTEGRALE DE LA CONFERENCE DE PRESSE DU GENERAL DE GAULLE . \ <p class=\\"media-body__summary\\">VERSION INTEGRALE DE LA CONFERENCE DE PRESSE DU GENERAL DE GAULLE . \
- PA le Pr\\u00e9sident DE GAULLE : il ouvre les bras et s'assied. DP journalis...<\/p>\\n\ - PA le Pr\\u00e9sident DE GAULLE : il ouvre les bras et s'assied. DP journalis...<\\/p>\\n\
<\/div>\\n<\/div><!-- \/.media -->\\n" <\\/div>\\n<\\/div><!-- \\/.media -->\\n"
} }
""" """
response = mock.Mock(text=json) response = mock.Mock(text=json)

2
tests/unit/engines/test_mediawiki.py
View File

@ -118,7 +118,7 @@ class TestMediawikiEngine(SearxTestCase):
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
self.assertEqual(len(results), 0) self.assertEqual(len(results), 0)
json = """ json = r"""
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"http:\/\/www.mediawiki.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.mediawiki.com\/artist\/1217","type":"artist"}

2
tests/unit/engines/test_mixcloud.py
View File

@ -55,7 +55,7 @@ class TestMixcloudEngine(SearxTestCase):
self.assertEqual(results[0]['content'], 'User') self.assertEqual(results[0]['content'], 'User')
self.assertTrue('http://www.mixcloud.com/user/this-is-the-url/' in results[0]['embedded']) self.assertTrue('http://www.mixcloud.com/user/this-is-the-url/' in results[0]['embedded'])
json = """ json = r"""
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"http:\/\/www.mixcloud.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.mixcloud.com\/artist\/1217","type":"artist"}

2
tests/unit/engines/test_searchcode_code.py
View File

@ -63,7 +63,7 @@ class TestSearchcodeCodeEngine(SearxTestCase):
self.assertEqual(results[0]['repository'], 'https://repo') self.assertEqual(results[0]['repository'], 'https://repo')
self.assertEqual(results[0]['code_language'], 'cpp') self.assertEqual(results[0]['code_language'], 'cpp')
json = """ json = r"""
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"http:\/\/www.searchcode_code.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.searchcode_code.com\/artist\/1217","type":"artist"}

2
tests/unit/engines/test_searchcode_doc.py
View File

@ -61,7 +61,7 @@ class TestSearchcodeDocEngine(SearxTestCase):
self.assertIn('test', results[0]['content']) self.assertIn('test', results[0]['content'])
self.assertIn('Description', results[0]['content']) self.assertIn('Description', results[0]['content'])
json = """ json = r"""
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",
"link":"http:\/\/www.searchcode_doc.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.searchcode_doc.com\/artist\/1217","type":"artist"}

6
tests/unit/engines/test_wolframalpha_noapi.py
View File

@ -28,7 +28,7 @@ class TestWolframAlphaNoAPIEngine(SearxTestCase):
request = Request(headers={'Referer': referer_url}) request = Request(headers={'Referer': referer_url})
# test failure # test failure
json = ''' json = r'''
{"queryresult" : { {"queryresult" : {
"success" : false, "success" : false,
"error" : false, "error" : false,
@ -42,7 +42,7 @@ class TestWolframAlphaNoAPIEngine(SearxTestCase):
self.assertEqual(wolframalpha_noapi.response(response), []) self.assertEqual(wolframalpha_noapi.response(response), [])
# test basic case # test basic case
json = ''' json = r'''
{"queryresult" : { {"queryresult" : {
"success" : true, "success" : true,
"error" : false, "error" : false,
@ -143,7 +143,7 @@ class TestWolframAlphaNoAPIEngine(SearxTestCase):
self.assertEqual('Wolfram|Alpha', results[1]['title']) self.assertEqual('Wolfram|Alpha', results[1]['title'])
# test calc # test calc
json = """ json = r"""
{"queryresult" : { {"queryresult" : {
"success" : true, "success" : true,
"error" : false, "error" : false,