From 439cf0559ad843f1550a50fd840276ae7cff0af4 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Wed, 9 Dec 2015 01:20:46 +0100 Subject: [PATCH] [fix] replace the dead google images ajax api with a working one --- searx/engines/google_images.py | 59 ++++++------ searx/tests/engines/test_google_images.py | 111 ++++++---------------- 2 files changed, 57 insertions(+), 113 deletions(-) diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 85963a16f..ca067e14f 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -2,41 +2,42 @@ Google (Images) @website https://www.google.com - @provide-api yes (https://developers.google.com/web-search/docs/), - deprecated! + @provide-api yes (https://developers.google.com/custom-search/) @using-api yes - @results JSON - @stable yes (but deprecated) + @results HTML chunk + @stable no @parse url, title, img_src """ -from urllib import urlencode, unquote +from urllib import urlencode +from urlparse import parse_qs from json import loads +from lxml import html # engine dependent config categories = ['images'] paging = True safesearch = True -# search-url -url = 'https://ajax.googleapis.com/' -search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe={safesearch}&filter=off&{query}' +search_url = 'https://www.google.com/search'\ + '?{query}'\ + '&tbm=isch'\ + '&ijn=1'\ + '&start={offset}' # do search-request def request(query, params): - offset = (params['pageno'] - 1) * 8 - - if params['safesearch'] == 0: - safesearch = 'off' - else: - safesearch = 'on' + offset = (params['pageno'] - 1) * 100 params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset, safesearch=safesearch) + if safesearch and params['safesearch']: + params['url'] += '&' + urlencode({'safe': 'active'}) + return params @@ -44,30 +45,28 @@ def request(query, params): def response(resp): results = [] - search_res = loads(resp.text) - - # return empty array if there are no results - if not search_res.get('responseData', {}).get('results'): - return [] + dom = html.fromstring(resp.text) # parse results - for result in search_res['responseData']['results']: - href = result['originalContextUrl'] - title = result['title'] - if 'url' not in result: - continue - thumbnail_src = result['tbUrl'] + for result in dom.xpath('//div[@data-ved]'): + data_url = result.xpath('./a/@href')[0] + data_query = {k: v[0] for k, v in parse_qs(data_url.split('?', 1)[1]).iteritems()} + + metadata = loads(result.xpath('./div[@class="rg_meta"]/text()')[0]) + + thumbnail_src = metadata['tu'] # http to https thumbnail_src = thumbnail_src.replace("http://", "https://") # append result - results.append({'url': href, - 'title': title, - 'content': result['content'], - 'thumbnail_src': thumbnail_src, - 'img_src': unquote(result['url']), + results.append({'url': data_query['imgrefurl'], + 'title': metadata['pt'], + 'content': metadata['s'], + 'thumbnail_src': metadata['tu'], + 'img_src': data_query['imgurl'], 'template': 'images.html'}) # return results + print len(results) return results diff --git a/searx/tests/engines/test_google_images.py b/searx/tests/engines/test_google_images.py index 9bef692d4..876d0af1e 100644 --- a/searx/tests/engines/test_google_images.py +++ b/searx/tests/engines/test_google_images.py @@ -10,15 +10,15 @@ class TestGoogleImagesEngine(SearxTestCase): query = 'test_query' dicto = defaultdict(dict) dicto['pageno'] = 1 + dicto['safesearch'] = 1 params = google_images.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) - self.assertIn('googleapis.com', params['url']) - self.assertIn('safe=on', params['url']) + self.assertIn('safe=active', params['url']) dicto['safesearch'] = 0 params = google_images.request(query, dicto) - self.assertIn('safe=off', params['url']) + self.assertNotIn('safe', params['url']) def test_response(self): self.assertRaises(AttributeError, google_images.response, None) @@ -26,88 +26,33 @@ class TestGoogleImagesEngine(SearxTestCase): self.assertRaises(AttributeError, google_images.response, '') self.assertRaises(AttributeError, google_images.response, '[]') - response = mock.Mock(text='{}') + response = mock.Mock(text='
') self.assertEqual(google_images.response(response), []) - response = mock.Mock(text='{"data": []}') - self.assertEqual(google_images.response(response), []) - - json = """ - { - "responseData": { - "results": [ - { - "GsearchResultClass": "GimageSearch", - "width": "400", - "height": "400", - "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ", - "tbWidth": "124", - "tbHeight": "124", - "unescapedUrl": "http://unescaped.url.jpg", - "url": "http://image.url.jpg", - "visibleUrl": "insolitebuzz.fr", - "title": "This is the title", - "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire", - "originalContextUrl": "http://this.is.the.url", - "content": "test", - "contentNoFormatting": "test", - "tbUrl": "http://thumbnail.url" - } - ] - }, - "responseDetails": null, - "responseStatus": 200 - } - """ - response = mock.Mock(text=json) + html = """ +
+ +
+
+ Image result for south +
+
504 × 598 - clker.com +
+
+ +
+ {"id":"bQWQ9wz9loJmjM:","isu":"clker.com","ity":"png","md":"/search?tbs\u003dsbi:AMhZZit7u1mHyop9pQisu-5idR-8W_1Itvwc3afChmsjQYPx_1yYMzBvUZgtkcGoojqekKZ-6n_1rjX9ySH0OWA_1eO5OijFY6BBDw_1GApr6xxb1bXJcBcj-DiguMoXWW7cZSG7MRQbwnI5SoDZNXcv_1xGszy886I7NVb_1oRKSliTHtzqbXAxhvYreM","msu":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAQSEgltBZD3DP2WgiG-U42R4G0RFw","oh":598,"os":"13KB","ow":504,"pt":"South Arrow Clip Art at Clker.com - vector clip art online ...","rid":"vlONkeBtERfDuM","s":"Download this image as:","sc":1,"si":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAESEgltBZD3DP2WgiG-U42R4G0RFw","th":245,"tu":"https://thumbnail.url/","tw":206} +
+
+
+
+ """ # noqa + response = mock.Mock(text=html) results = google_images.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'This is the title') - self.assertEqual(results[0]['url'], 'http://this.is.the.url') - self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url') - self.assertEqual(results[0]['img_src'], 'http://image.url.jpg') - self.assertEqual(results[0]['content'], 'test') - - json = """ - { - "responseData": { - "results": [ - { - "GsearchResultClass": "GimageSearch", - "width": "400", - "height": "400", - "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ", - "tbWidth": "124", - "tbHeight": "124", - "unescapedUrl": "http://unescaped.url.jpg", - "visibleUrl": "insolitebuzz.fr", - "title": "This is the title", - "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire", - "originalContextUrl": "http://this.is.the.url", - "content": "test", - "contentNoFormatting": "test", - "tbUrl": "http://thumbnail.url" - } - ] - }, - "responseDetails": null, - "responseStatus": 200 - } - """ - response = mock.Mock(text=json) - results = google_images.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) - - json = """ - { - "responseData": {}, - "responseDetails": null, - "responseStatus": 200 - } - """ - response = mock.Mock(text=json) - results = google_images.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) + self.assertEqual(results[0]['title'], u'South Arrow Clip Art at Clker.com - vector clip art online ...') + self.assertEqual(results[0]['url'], 'http://www.clker.com/clipart-south-arrow.html') + self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url/') + self.assertEqual(results[0]['img_src'], 'http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png') + self.assertEqual(results[0]['content'], 'Download this image as:')