kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-10 15:10:10 +01:00
Code

initial xpath work (WIP)

Browse Source
This commit is contained in:
Austin-Olacsi 2024-06-04 02:21:28 -06:00 committed by GitHub
parent 21b0926c50
commit 6bfb101e6e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
searx/engines/yandex.py
View File

@ -7,6 +7,7 @@ from urllib.parse import urlencode, urlparse, parse_qs
from lxml import html
from html import unescape
from searx import logger
from searx import utils
from searx.exceptions import SearxEngineCaptchaException
from datetime import datetime
@ -31,6 +32,10 @@ base_url_videos = 'https://yandex.com/video/search'
url_extension = 'tmpl_version=releases%2Ffrontend%2Fvideo%2Fv1.1168.0%238d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63'
url_xpath = "//a[@class='b-serp-item__title-link']/@href"
title_xpath = "//h3[@class='b-serp-item__title']/a[@class='b-serp-item__title-link']/span"
content_xpath = "//div[@class='b-serp-item__content']//div[@class='b-serp-item__text']"
images_request_block = '{"blocks":[{"block":"extra-content","params":{},"version":2},{"block":"i-global__params:ajax","params":{},"version":2},{"block":"search2:ajax","params":{},"version":2},{"block":"preview__isWallpaper","params":{},"version":2},{"block":"content_type_search","params":{},"version":2},{"block":"serp-controller","params":{},"version":2},{"block":"cookies_ajax","params":{},"version":2},{"block":"advanced-search-block","params":{},"version":2}],"metadata":{"bundles":{"lb":"AS?(E<X120"},"assets":{"las":"justifier-height=1;justifier-setheight=1;fitimages-height=1;justifier-fitincuts=1;react-with-dom=1;"},"extraContent":{"names":["i-react-ajax-adapter"]}}}'
videos_request_block = '{"blocks":[{"block":"extra-content","params":{},"version":2},{"block":"i-global__params:ajax","params":{},"version":2},{"block":"search2:ajax","params":{},"version":2},{"block":"vital-incut","params":{},"version":2},{"block":"content_type_search","params":{},"version":2},{"block":"serp-controller","params":{},"version":2},{"block":"cookies_ajax","params":{},"version":2}],"metadata":{"bundles":{"lb":"^G]!q<X120"},"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},"extraContent":{"names":["i-react-ajax-adapter"]}}}'
@ -103,27 +108,29 @@ def get_youtube_iframe_src(url):
if video_id:
return 'https://www.youtube-nocookie.com/embed/' + video_id[0]
def response(resp):
if yandex_category == 'web':
if (resp.url).path.startswith('/showcaptcha'):
raise SearxEngineCaptchaException()
html_data = html.fromstring(resp.text)
text = html.tostring(html_data, encoding='unicode')
dom = html.fromstring(resp.text)
results_dom = dom.xpath('//li[contains(@class, "serp-item")]')
urls = re.findall(r'title-link" href="(.*?)" target="_blank"', text)
titles = re.findall(r'tabindex="4"><span>(.*?)</span></a></h3>', text)
contents = re.findall(r'"b-serp-item__text">(.*?)</div>', text)
results = []
for result_dom in results_dom:
urls = result_dom.xpath(url_xpath)
titles = result_dom.xpath(title_xpath)
contents = result_dom.xpath(content_xpath)
results = [
{
"url": url,
"title": title,
"content": content,
}
for url, title, content in zip(urls, titles, contents)
]
title_texts = [title.xpath("normalize-space(.)") for title in titles]
content_texts = [content.xpath("normalize-space(.)") for content in contents]
for url, title_text, content_text in zip(urls, title_texts, content_texts):
results.append({
"url": url,
"title": title_text,
"content": content_text,
})
return results