# SPDX-License-Identifier: AGPL-3.0-or-later
"""Yahoo Search (Web)
Languages are supported by mapping the language to a domain. If domain is not
found in :py:obj:`lang2domain` URL ``<lang>.search.yahoo.com`` is used.
"""
from urllib.parse import (
unquote,
urlencode,
)
from lxml import html
from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
extract_text,
html_to_text,
)
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
# about
about = {
"website": 'https://search.yahoo.com/',
"wikidata_id": None,
"official_api_documentation": 'https://developer.yahoo.com/api/',
"use_official_api": False,
"require_api_key": False,
"results": 'HTML',
}
# engine dependent config
categories = ['general', 'web']
paging = True
time_range_support = True
# send_accept_language_header = True
time_range_dict = {
'day': ('1d', 'd'),
'week': ('1w', 'w'),
'month': ('1m', 'm'),
}
region2domain = {
"CO": "co.search.yahoo.com", # Colombia
"TH": "th.search.yahoo.com", # Thailand
"VE": "ve.search.yahoo.com", # Venezuela
"CL": "cl.search.yahoo.com", # Chile
"HK": "hk.search.yahoo.com", # Hong Kong
"PE": "pe.search.yahoo.com", # Peru
"CA": "ca.search.yahoo.com", # Canada
"DE": "de.search.yahoo.com", # Germany
"FR": "fr.search.yahoo.com", # France
"TW": "tw.search.yahoo.com", # Taiwan
"GB": "uk.search.yahoo.com", # United Kingdom
"UK": "uk.search.yahoo.com",
"BR": "br.search.yahoo.com", # Brazil
"IN": "in.search.yahoo.com", # India
"ES": "espanol.search.yahoo.com", # Espanol
"PH": "ph.search.yahoo.com", # Philippines
"AR": "ar.search.yahoo.com", # Argentina
"MX": "mx.search.yahoo.com", # Mexico
"SG": "sg.search.yahoo.com", # Singapore
}
"""Map regions to domain"""
lang2domain = {
'zh_chs': 'hk.search.yahoo.com',
'zh_cht': 'tw.search.yahoo.com',
'any': 'search.yahoo.com',
'en': 'search.yahoo.com',
'bg': 'search.yahoo.com',
'cs': 'search.yahoo.com',
'da': 'search.yahoo.com',
'el': 'search.yahoo.com',
'et': 'search.yahoo.com',
'he': 'search.yahoo.com',
'hr': 'search.yahoo.com',
'ja': 'search.yahoo.com',
'ko': 'search.yahoo.com',
'sk': 'search.yahoo.com',
'sl': 'search.yahoo.com',
}
"""Map language to domain"""
yahoo_languages = {
"all": "any",
"ar": "ar", # Arabic
"bg": "bg", # Bulgarian
"cs": "cs", # Czech
"da": "da", # Danish
"de": "de", # German
"el": "el", # Greek
"en": "en", # English
"es": "es", # Spanish
"et": "et", # Estonian
"fi": "fi", # Finnish
"fr": "fr", # French
"he": "he", # Hebrew
"hr": "hr", # Croatian
"hu": "hu", # Hungarian
"it": "it", # Italian
"ja": "ja", # Japanese
"ko": "ko", # Korean
"lt": "lt", # Lithuanian
"lv": "lv", # Latvian
"nl": "nl", # Dutch
"no": "no", # Norwegian
"pl": "pl", # Polish
"pt": "pt", # Portuguese
"ro": "ro", # Romanian
"ru": "ru", # Russian
"sk": "sk", # Slovak
"sl": "sl", # Slovenian
"sv": "sv", # Swedish
"th": "th", # Thai
"tr": "tr", # Turkish
"zh": "zh_chs", # Chinese (Simplified)
"zh_Hans": "zh_chs",
'zh-CN': "zh_chs",
"zh_Hant": "zh_cht", # Chinese (Traditional)
"zh-HK": "zh_cht",
'zh-TW': "zh_cht",
}
[docs]
def request(query, params):
"""build request"""
lang, region = (params["language"].split("-") + [None])[:2]
lang = yahoo_languages.get(lang, "any")
offset = (params['pageno'] - 1) * 7 + 1
age, btf = time_range_dict.get(params['time_range'], ('', ''))
args = urlencode(
{
'p': query,
'ei': 'UTF-8',
'fl': 1,
'vl': 'lang_' + lang,
'btf': btf,
'fr2': 'time',
'age': age,
'b': offset,
'xargs': 0,
}
)
domain = region2domain.get(region)
if not domain:
domain = lang2domain.get(lang, '%s.search.yahoo.com' % lang)
params['url'] = 'https://%s/search?%s' % (domain, args)
params['domain'] = domain
[docs]
def parse_url(url_string):
"""remove yahoo-specific tracking-url"""
endings = ['/RS', '/RK']
endpositions = []
start = url_string.find('http', url_string.find('/RU=') + 1)
for ending in endings:
endpos = url_string.rfind(ending)
if endpos > -1:
endpositions.append(endpos)
if start == 0 or len(endpositions) == 0:
return url_string
end = min(endpositions)
return unquote(url_string[start:end])
[docs]
def response(resp):
"""parse response"""
results = []
dom = html.fromstring(resp.text)
url_xpath = './/div[contains(@class,"compTitle")]/h3/a/@href'
title_xpath = './/h3//a/@aria-label'
domain = resp.search_params['domain']
if domain == "search.yahoo.com":
url_xpath = './/div[contains(@class,"compTitle")]/a/@href'
title_xpath = './/div[contains(@class,"compTitle")]/a/h3/span'
# parse results
for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'):
url = eval_xpath_getindex(result, url_xpath, 0, default=None)
if url is None:
continue
url = parse_url(url)
title = eval_xpath_getindex(result, title_xpath, 0, default='')
title: str = extract_text(title)
content = eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0, default='')
content: str = extract_text(content, allow_none=True)
# append result
results.append(
{
'url': url,
# title sometimes contains HTML tags / see
# https://github.com/searxng/searxng/issues/3790
'title': " ".join(html_to_text(title).strip().split()),
'content': " ".join(html_to_text(content).strip().split()),
}
)
for suggestion in eval_xpath_list(dom, '//div[contains(@class, "AlsoTry")]//table//a'):
# append suggestion
results.append({'suggestion': extract_text(suggestion)})
return results
