kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-19 02:40:11 +01:00
Code

[fix] replace the dead google images ajax api with a working one

Browse Source
This commit is contained in:
Adam Tauber 2015-12-09 01:20:46 +01:00
parent 911ed7987c
commit 439cf0559a
2 changed files with 57 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
searx/engines/google_images.py
View File

@ -2,41 +2,42 @@
Google (Images) Google (Images)
@website https://www.google.com @website https://www.google.com
@provide-api yes (https://developers.google.com/web-search/docs/), @provide-api yes (https://developers.google.com/custom-search/)
deprecated!
@using-api yes @using-api yes
@results JSON @results HTML chunk
@stable yes (but deprecated) @stable no
@parse url, title, img_src @parse url, title, img_src
""" """
from urllib import urlencode, unquote from urllib import urlencode
from urlparse import parse_qs
from json import loads from json import loads
from lxml import html
# engine dependent config # engine dependent config
categories = ['images'] categories = ['images']
paging = True paging = True
safesearch = True safesearch = True
# search-url search_url = 'https://www.google.com/search'\
url = 'https://ajax.googleapis.com/' '?{query}'\
search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe={safesearch}&filter=off&{query}' '&tbm=isch'\
'&ijn=1'\
'&start={offset}'
# do search-request # do search-request
def request(query, params): def request(query, params):
offset = (params['pageno'] - 1) * 8 offset = (params['pageno'] - 1) * 100
if params['safesearch'] == 0:
safesearch = 'off'
else:
safesearch = 'on'
params['url'] = search_url.format(query=urlencode({'q': query}), params['url'] = search_url.format(query=urlencode({'q': query}),
offset=offset, offset=offset,
safesearch=safesearch) safesearch=safesearch)
if safesearch and params['safesearch']:
params['url'] += '&' + urlencode({'safe': 'active'})
return params return params
@ -44,30 +45,28 @@ def request(query, params):
def response(resp): def response(resp):
results = [] results = []
search_res = loads(resp.text) dom = html.fromstring(resp.text)
# return empty array if there are no results
if not search_res.get('responseData', {}).get('results'):
return []
# parse results # parse results
for result in search_res['responseData']['results']: for result in dom.xpath('//div[@data-ved]'):
href = result['originalContextUrl'] data_url = result.xpath('./a/@href')[0]
title = result['title'] data_query = {k: v[0] for k, v in parse_qs(data_url.split('?', 1)[1]).iteritems()}
if 'url' not in result:
continue metadata = loads(result.xpath('./div[@class="rg_meta"]/text()')[0])
thumbnail_src = result['tbUrl']
thumbnail_src = metadata['tu']
# http to https # http to https
thumbnail_src = thumbnail_src.replace("http://", "https://") thumbnail_src = thumbnail_src.replace("http://", "https://")
# append result # append result
results.append({'url': href, results.append({'url': data_query['imgrefurl'],
'title': title, 'title': metadata['pt'],
'content': result['content'], 'content': metadata['s'],
'thumbnail_src': thumbnail_src, 'thumbnail_src': metadata['tu'],
'img_src': unquote(result['url']), 'img_src': data_query['imgurl'],
'template': 'images.html'}) 'template': 'images.html'})
# return results # return results
print len(results)
return results return results

111
searx/tests/engines/test_google_images.py
View File

@ -10,15 +10,15 @@ class TestGoogleImagesEngine(SearxTestCase):
query = 'test_query' query = 'test_query'
dicto = defaultdict(dict) dicto = defaultdict(dict)
dicto['pageno'] = 1 dicto['pageno'] = 1
dicto['safesearch'] = 1
params = google_images.request(query, dicto) params = google_images.request(query, dicto)
self.assertIn('url', params) self.assertIn('url', params)
self.assertIn(query, params['url']) self.assertIn(query, params['url'])
self.assertIn('googleapis.com', params['url']) self.assertIn('safe=active', params['url'])
self.assertIn('safe=on', params['url'])
dicto['safesearch'] = 0 dicto['safesearch'] = 0
params = google_images.request(query, dicto) params = google_images.request(query, dicto)
self.assertIn('safe=off', params['url']) self.assertNotIn('safe', params['url'])
def test_response(self): def test_response(self):
self.assertRaises(AttributeError, google_images.response, None) self.assertRaises(AttributeError, google_images.response, None)
@ -26,88 +26,33 @@ class TestGoogleImagesEngine(SearxTestCase):
self.assertRaises(AttributeError, google_images.response, '') self.assertRaises(AttributeError, google_images.response, '')
self.assertRaises(AttributeError, google_images.response, '[]') self.assertRaises(AttributeError, google_images.response, '[]')
response = mock.Mock(text='{}') response = mock.Mock(text='<div></div>')
self.assertEqual(google_images.response(response), []) self.assertEqual(google_images.response(response), [])
response = mock.Mock(text='{"data": []}') html = """
self.assertEqual(google_images.response(response), []) <div style="display:none">
<div eid="fWhnVq4Shqpp3pWo4AM" id="isr_scm_1" style="display:none"></div>
json = """ <div data-cei="fWhnVq4Shqpp3pWo4AM" class="rg_add_chunk"><!--m-->
{ <div class="rg_di rg_el ivg-i" data-ved="0ahUKEwjuxPWQts3JAhUGVRoKHd4KCjwQMwgDKAAwAA">
"responseData": { <a href="/imgres?imgurl=http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png&amp;imgrefurl=http://www.clker.com/clipart-south-arrow.html&amp;h=598&amp;w=504&amp;tbnid=bQWQ9wz9loJmjM:&amp;docid=vlONkeBtERfDuM&amp;ei=fWhnVq4Shqpp3pWo4AM&amp;tbm=isch" jsaction="fire.ivg_o;mouseover:str.hmov;mouseout:str.hmou" class="rg_l"><img data-src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRsxy3gKnEX0lrwwpRxdPWyLJ8iZ--PXZ-ThbBA2_xXDG_bdQutMQ" data-sz="f" name="bQWQ9wz9loJmjM:" class="rg_i" alt="Image result for south" jsaction="load:str.tbn" onload="google.aft&&google.aft(this)">
"results": [ <div class="_aOd rg_ilm">
{ <div class="rg_ilmbg"><span class="rg_ilmn"> 504&nbsp;&#215;&nbsp;598 - clker.com </span>
"GsearchResultClass": "GimageSearch", </div>
"width": "400", </div>
"height": "400", </a>
"imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ", <div class="rg_meta">
"tbWidth": "124", {"id":"bQWQ9wz9loJmjM:","isu":"clker.com","ity":"png","md":"/search?tbs\u003dsbi:AMhZZit7u1mHyop9pQisu-5idR-8W_1Itvwc3afChmsjQYPx_1yYMzBvUZgtkcGoojqekKZ-6n_1rjX9ySH0OWA_1eO5OijFY6BBDw_1GApr6xxb1bXJcBcj-DiguMoXWW7cZSG7MRQbwnI5SoDZNXcv_1xGszy886I7NVb_1oRKSliTHtzqbXAxhvYreM","msu":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAQSEgltBZD3DP2WgiG-U42R4G0RFw","oh":598,"os":"13KB","ow":504,"pt":"South Arrow Clip Art at Clker.com - vector clip art online ...","rid":"vlONkeBtERfDuM","s":"Download this image as:","sc":1,"si":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAESEgltBZD3DP2WgiG-U42R4G0RFw","th":245,"tu":"https://thumbnail.url/","tw":206}
"tbHeight": "124", </div>
"unescapedUrl": "http://unescaped.url.jpg", </div><!--n--><!--m-->
"url": "http://image.url.jpg", </div>
"visibleUrl": "insolitebuzz.fr", </div>
"title": "This is the title", """ # noqa
"titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire", response = mock.Mock(text=html)
"originalContextUrl": "http://this.is.the.url",
"content": "<b>test</b>",
"contentNoFormatting": "test",
"tbUrl": "http://thumbnail.url"
}
]
},
"responseDetails": null,
"responseStatus": 200
}
"""
response = mock.Mock(text=json)
results = google_images.response(response) results = google_images.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
self.assertEqual(len(results), 1) self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the title') self.assertEqual(results[0]['title'], u'South Arrow Clip Art at Clker.com - vector clip art online ...')
self.assertEqual(results[0]['url'], 'http://this.is.the.url') self.assertEqual(results[0]['url'], 'http://www.clker.com/clipart-south-arrow.html')
self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url') self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url/')
self.assertEqual(results[0]['img_src'], 'http://image.url.jpg') self.assertEqual(results[0]['img_src'], 'http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png')
self.assertEqual(results[0]['content'], '<b>test</b>') self.assertEqual(results[0]['content'], 'Download this image as:')
json = """
{
"responseData": {
"results": [
{
"GsearchResultClass": "GimageSearch",
"width": "400",
"height": "400",
"imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ",
"tbWidth": "124",
"tbHeight": "124",
"unescapedUrl": "http://unescaped.url.jpg",
"visibleUrl": "insolitebuzz.fr",
"title": "This is the title",
"titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire",
"originalContextUrl": "http://this.is.the.url",
"content": "<b>test</b>",
"contentNoFormatting": "test",
"tbUrl": "http://thumbnail.url"
}
]
},
"responseDetails": null,
"responseStatus": 200
}
"""
response = mock.Mock(text=json)
results = google_images.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
json = """
{
"responseData": {},
"responseDetails": null,
"responseStatus": 200
}
"""
response = mock.Mock(text=json)
results = google_images.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)