mirror of
https://github.com/searxng/searxng.git
synced 2024-11-05 04:40:11 +01:00
Merge pull request #676 from return42/fix-bing-lang
Fix issues on running update_languages.py
This commit is contained in:
commit
21d7c8b367
@ -193,7 +193,7 @@ def set_language_attributes(engine):
|
||||
if hasattr(engine, '_fetch_supported_languages'):
|
||||
headers = {
|
||||
'User-Agent': gen_useragent(),
|
||||
'Accept-Language': 'ja-JP,ja;q=0.8,en-US;q=0.5,en;q=0.3', # bing needs a non-English language
|
||||
'Accept-Language': "en-US,en;q=0.5", # bing needs to set the English language
|
||||
}
|
||||
engine.fetch_supported_languages = (
|
||||
# pylint: disable=protected-access
|
||||
|
@ -6,7 +6,7 @@
|
||||
"""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlencode
|
||||
from urllib.parse import urlencode, urlparse, parse_qs
|
||||
from lxml import html
|
||||
from searx.utils import eval_xpath, extract_text, match_language
|
||||
|
||||
@ -25,7 +25,7 @@ paging = True
|
||||
time_range_support = False
|
||||
safesearch = False
|
||||
supported_languages_url = 'https://www.bing.com/account/general'
|
||||
language_aliases = {'zh-CN': 'zh-CHS', 'zh-TW': 'zh-CHT', 'zh-HK': 'zh-CHT'}
|
||||
language_aliases = {}
|
||||
|
||||
# search-url
|
||||
base_url = 'https://www.bing.com/'
|
||||
@ -127,18 +127,27 @@ def response(resp):
|
||||
|
||||
# get supported languages from their site
|
||||
def _fetch_supported_languages(resp):
|
||||
|
||||
lang_tags = set()
|
||||
|
||||
setmkt = re.compile('setmkt=([^&]*)')
|
||||
dom = html.fromstring(resp.text)
|
||||
lang_links = eval_xpath(dom, "//li/a[contains(@href, 'setmkt')]")
|
||||
lang_links = eval_xpath(dom, '//div[@id="language-section"]//li')
|
||||
|
||||
for a in lang_links:
|
||||
href = eval_xpath(a, './@href')[0]
|
||||
match = setmkt.search(href)
|
||||
l_tag = match.groups()[0]
|
||||
_lang, _nation = l_tag.split('-', 1)
|
||||
l_tag = _lang.lower() + '-' + _nation.upper()
|
||||
lang_tags.add(l_tag)
|
||||
for _li in lang_links:
|
||||
|
||||
href = eval_xpath(_li, './/@href')[0]
|
||||
(_scheme, _netloc, _path, _params, query, _fragment) = urlparse(href)
|
||||
query = parse_qs(query, keep_blank_values=True)
|
||||
|
||||
# fmt: off
|
||||
setlang = query.get('setlang', [None, ])[0]
|
||||
# example: 'mn-Cyrl-MN' --> '['mn', 'Cyrl-MN']
|
||||
lang, nation = (setlang.split('-', maxsplit=1) + [None,])[:2] # fmt: skip
|
||||
# fmt: on
|
||||
|
||||
if not nation:
|
||||
nation = lang.upper()
|
||||
tag = lang + '-' + nation
|
||||
lang_tags.add(tag)
|
||||
|
||||
return list(lang_tags)
|
||||
|
@ -35,6 +35,8 @@ def fetch_supported_languages():
|
||||
if type(engines_languages[engine_name]) == list:
|
||||
engines_languages[engine_name] = sorted(engines_languages[engine_name])
|
||||
|
||||
print("fetched languages from %s engines" % len(engines_languages))
|
||||
|
||||
# write json file
|
||||
with open(engines_languages_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(engines_languages, f, indent=2, sort_keys=True)
|
||||
@ -97,7 +99,11 @@ def join_language_lists(engines_languages):
|
||||
country_name = ''
|
||||
if locale:
|
||||
# get country name from babel's Locale object
|
||||
country_name = locale.get_territory_name()
|
||||
try:
|
||||
country_name = locale.get_territory_name()
|
||||
except FileNotFoundError as exc:
|
||||
print("ERROR: %s --> %s" % (locale, exc))
|
||||
locale = None
|
||||
|
||||
language_list[short_code]['countries'][lang_code] = {'country_name': country_name, 'counter': set()}
|
||||
|
||||
@ -186,17 +192,24 @@ def write_languages_file(languages):
|
||||
"language_codes =",
|
||||
)
|
||||
|
||||
language_codes = tuple(
|
||||
[
|
||||
(
|
||||
code,
|
||||
languages[code]['name'].split(' (')[0],
|
||||
languages[code].get('country_name') or '',
|
||||
languages[code].get('english_name') or '',
|
||||
)
|
||||
for code in sorted(languages)
|
||||
]
|
||||
)
|
||||
language_codes = []
|
||||
|
||||
for code in sorted(languages):
|
||||
|
||||
name = languages[code]['name']
|
||||
if name is None:
|
||||
print("ERROR: languages['%s'] --> %s" % (code, languages[code]))
|
||||
continue
|
||||
item = (
|
||||
code,
|
||||
languages[code]['name'].split(' (')[0],
|
||||
languages[code].get('country_name') or '',
|
||||
languages[code].get('english_name') or '',
|
||||
)
|
||||
|
||||
language_codes.append(item)
|
||||
|
||||
language_codes = tuple(language_codes)
|
||||
|
||||
with open(languages_file, 'w') as new_file:
|
||||
file_content = "{file_headers} \\\n{language_codes}".format(
|
||||
|
Loading…
Reference in New Issue
Block a user