fix fetch_langauges to be more accurate

Add languages supported by either all default general engines or 10 engines.

searx/engines/duckduckgo.py

@@ -23,7 +23,7 @@ from searx.url_utils import urlencode
 categories = ['general']
 paging = True
 language_support = True
-supported_languages_url = 'https://duckduckgo.com/d2030.js'
+supported_languages_url = 'https://duckduckgo.com/util/u172.js'
 time_range_support = True
 
 # search-url

searx/engines/google.py

@@ -72,7 +72,7 @@ country_to_hostname = {
     'RO': 'www.google.ro',  # Romania
     'RU': 'www.google.ru',  # Russia
     'SK': 'www.google.sk',  # Slovakia
-    'SL': 'www.google.si',  # Slovenia (SL -> si)
+    'SI': 'www.google.si',  # Slovenia
     'SE': 'www.google.se',  # Sweden
     'TH': 'www.google.co.th',  # Thailand
     'TR': 'www.google.com.tr',  # Turkey

searx/languages.py

@@ -5,11 +5,7 @@
 language_codes = (
     (u"ar-SA", u"العربية", u"", u"Arabic"),
     (u"bg-BG", u"Български", u"", u"Bulgarian"),
-    (u"ca", u"Català", u"", u"Catalan"),
-    (u"ca-AD", u"Català", u"Andorra", u"Catalan"),
-    (u"ca-CT", u"Català", u"", u"Catalan"),
-    (u"ca-ES", u"Català", u"Espanya", u"Catalan"),
-    (u"ca-FR", u"Català", u"França", u"Catalan"),
+    (u"ca-ES", u"Català", u"", u"Catalan"),
     (u"cs-CZ", u"Čeština", u"", u"Czech"),
     (u"da-DK", u"Dansk", u"", u"Danish"),
     (u"de", u"Deutsch", u"", u"German"),
@@ -21,55 +17,51 @@ language_codes = (
     (u"en-AU", u"English", u"Australia", u"English"),
     (u"en-CA", u"English", u"Canada", u"English"),
     (u"en-GB", u"English", u"United Kingdom", u"English"),
-    (u"en-ID", u"English", u"Indonesia", u"English"),
-    (u"en-IE", u"English", u"Ireland", u"English"),
     (u"en-IN", u"English", u"India", u"English"),
     (u"en-MY", u"English", u"Malaysia", u"English"),
-    (u"en-NZ", u"English", u"New Zealand", u"English"),
-    (u"en-PH", u"English", u"Philippines", u"English"),
-    (u"en-SG", u"English", u"Singapore", u"English"),
     (u"en-US", u"English", u"United States", u"English"),
-    (u"en-ZA", u"English", u"South Africa", u"English"),
     (u"es", u"Español", u"", u"Spanish"),
-    (u"es-AD", u"Español", u"Andorra", u"Spanish"),
     (u"es-AR", u"Español", u"Argentina", u"Spanish"),
-    (u"es-CL", u"Español", u"Chile", u"Spanish"),
-    (u"es-CO", u"Español", u"Colombia", u"Spanish"),
     (u"es-ES", u"Español", u"España", u"Spanish"),
     (u"es-MX", u"Español", u"México", u"Spanish"),
-    (u"es-PE", u"Español", u"Perú", u"Spanish"),
-    (u"es-US", u"Español", u"Estados Unidos", u"Spanish"),
     (u"et-EE", u"Eesti", u"", u"Estonian"),
+    (u"fa-IR", u"فارسی", u"", u"Persian"),
     (u"fi-FI", u"Suomi", u"", u"Finnish"),
     (u"fr", u"Français", u"", u"French"),
-    (u"fr-AD", u"Français", u"Andorre", u"French"),
     (u"fr-BE", u"Français", u"Belgique", u"French"),
     (u"fr-CA", u"Français", u"Canada", u"French"),
     (u"fr-CH", u"Français", u"Suisse", u"French"),
     (u"fr-FR", u"Français", u"France", u"French"),
     (u"he-IL", u"עברית", u"", u"Hebrew"),
+    (u"hr-HR", u"Hrvatski", u"", u"Croatian"),
     (u"hu-HU", u"Magyar", u"", u"Hungarian"),
-    (u"it", u"Italiano", u"", u"Italian"),
-    (u"it-CH", u"Italiano", u"Svizzera", u"Italian"),
-    (u"it-IT", u"Italiano", u"Italia", u"Italian"),
+    (u"id-ID", u"Indonesia", u"", u"Indonesian"),
+    (u"is-IS", u"Íslenska", u"", u"Icelandic"),
+    (u"it-IT", u"Italiano", u"", u"Italian"),
     (u"ja-JP", u"日本語", u"", u"Japanese"),
     (u"ko-KR", u"한국어", u"", u"Korean"),
+    (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),
+    (u"lv-LV", u"Latviešu", u"", u"Latvian"),
+    (u"ms-MY", u"Bahasa Melayu", u"", u"Malay"),
+    (u"nb-NO", u"Norsk Bokmål", u"", u"Norwegian Bokmål"),
     (u"nl", u"Nederlands", u"", u"Dutch"),
     (u"nl-BE", u"Nederlands", u"België", u"Dutch"),
     (u"nl-NL", u"Nederlands", u"Nederland", u"Dutch"),
-    (u"no-NO", u"Norsk", u"", u"Norwegian"),
     (u"pl-PL", u"Polski", u"", u"Polish"),
     (u"pt", u"Português", u"", u"Portuguese"),
-    (u"pt-AD", u"Português", u"Andorra", u"Portuguese"),
     (u"pt-BR", u"Português", u"Brasil", u"Portuguese"),
     (u"pt-PT", u"Português", u"Portugal", u"Portuguese"),
     (u"ro-RO", u"Română", u"", u"Romanian"),
     (u"ru-RU", u"Русский", u"", u"Russian"),
+    (u"sk-SK", u"Slovenčina", u"", u"Slovak"),
+    (u"sl-SI", u"Slovenščina", u"", u"Slovenian"),
+    (u"sr-RS", u"Српски", u"", u"Serbian"),
     (u"sv-SE", u"Svenska", u"", u"Swedish"),
     (u"th-TH", u"ไทย", u"", u"Thai"),
     (u"tr-TR", u"Türkçe", u"", u"Turkish"),
+    (u"uk-UA", u"Українська", u"", u"Ukrainian"),
+    (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),
     (u"zh", u"中文", u"", u"Chinese"),
     (u"zh-CN", u"中文", u"中国", u"Chinese"),
-    (u"zh-HK", u"中文", u"香港", u"Chinese"),
-    (u"zh-TW", u"中文", u"台湾", u"Chinese")
+    (u"zh-TW", u"中文", u"台灣", u"Chinese")
 )

utils/fetch_languages.py

@@ -2,83 +2,40 @@
 
 # This script generates languages.py from intersecting each engine's supported languages.
 #
-# The country names are obtained from http://api.geonames.org which requires registering as a user.
-#
 # Output files (engines_languages.json and languages.py)
 # are written in current directory to avoid overwriting in case something goes wrong.
 
-from requests import get
-from lxml.html import fromstring
-from json import loads, dump
+from json import dump
 import io
 from sys import path
+from babel import Locale, UnknownLocaleError
+from babel.languages import get_global
+
 path.append('../searx')  # noqa
 from searx import settings
-from searx.url_utils import urlencode
 from searx.engines import initialize_engines, engines
 
-# Geonames API for country names.
-geonames_user = ''  # ADD USER NAME HERE
-country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}'
-
 # Output files.
 engines_languages_file = 'engines_languages.json'
 languages_file = 'languages.py'
 
-engines_languages = {}
-
-
-# To filter out invalid codes and dialects.
-def valid_code(lang_code):
-    # filter invalid codes
-    # sl-SL is technically not invalid, but still a mistake
-    invalid_codes = ['sl-SL', 'wt-WT', 'jw']
-    invalid_countries = ['UK', 'XA', 'XL']
-    if lang_code[:2] == 'xx'\
-       or lang_code in invalid_codes\
-       or lang_code[-2:] in invalid_countries\
-       or is_dialect(lang_code):
-        return False
-
-    return True
-
-
-# Language codes with any additional tags other than language and country.
-def is_dialect(lang_code):
-    lang_code = lang_code.split('-')
-    if len(lang_code) > 2 or len(lang_code[0]) > 3:
-        return True
-    if len(lang_code) == 2 and len(lang_code[1]) > 2:
-        return True
-
-    return False
-
-
-# Get country name in specified language.
-def get_country_name(locale):
-    if geonames_user is '':
-        return ''
-
-    locale = locale.split('-')
-    if len(locale) != 2:
-        return ''
-
-    url = country_names_url.format(parameters=urlencode({'lang': locale[0],
-                                                         'country': locale[1],
-                                                         'username': geonames_user}))
-    response = get(url)
-    json = loads(response.text)
-    content = json.get('geonames', None)
-    if content is None or len(content) != 1:
-        print("No country name found for " + locale[0] + "-" + locale[1])
-        return ''
-
-    return content[0].get('countryName', '')
+# custom fixes for non standard locale codes
+# sl-SL is technically not invalid, but still a mistake
+# TODO: move to respective engines
+locale_fixes = {
+    'sl-sl': 'sl-SI',
+    'ar-xa': 'ar-SA',
+    'es-xl': 'es-419',
+    'zh-chs': 'zh-Hans-CN',
+    'zh-cht': 'zh-Hant-TW',
+    'tzh-tw': 'zh-Hant-TW',
+    'tzh-hk': 'zh-Hant-HK'
+}
 
 
 # Fetchs supported languages for each engine and writes json file with those.
 def fetch_supported_languages():
-    initialize_engines(settings['engines'])
+    engines_languages = {}
     for engine_name in engines:
         if hasattr(engines[engine_name], 'fetch_supported_languages'):
             try:
@@ -90,81 +47,134 @@ def fetch_supported_languages():
     with io.open(engines_languages_file, "w", encoding="utf-8") as f:
         dump(engines_languages, f, ensure_ascii=False)
 
+    return engines_languages
+
+
+# Get babel Locale object from lang_code if possible.
+def get_locale(lang_code):
+    try:
+        locale = Locale.parse(lang_code, sep='-')
+        return locale
+    except (UnknownLocaleError, ValueError):
+        return None
+
+
+# Append engine_name to list of engines that support locale.
+def add_engine_counter(lang_code, engine_name, languages):
+    if lang_code in languages:
+        if 'counter' not in languages[lang_code]:
+            languages[lang_code]['counter'] = [engine_name]
+        elif engine_name not in languages[lang_code]['counter']:
+            languages[lang_code]['counter'].append(engine_name)
 
-# Join all language lists.
-# Iterate all languages supported by each engine.
-def join_language_lists():
-    global languages
-    # include wikipedia first for more accurate language names
-    languages = {code: lang for code, lang
-                 in engines_languages['wikipedia'].items()
-                 if valid_code(code)}
 
+# Join all language lists.
+# TODO: Add language names from engine's language list if name not known by babel.
+def join_language_lists(engines_languages):
+    language_list = {}
     for engine_name in engines_languages:
-        for locale in engines_languages[engine_name]:
-            if valid_code(locale):
-                # if language is not on list or if it has no name yet
-                if locale not in languages or not languages[locale].get('name'):
-                    if isinstance(engines_languages[engine_name], dict):
-                        languages[locale] = engines_languages[engine_name][locale]
-                    else:
-                        languages[locale] = {}
-
-            # add to counter of engines that support given language
-            lang = locale.split('-')[0]
-            if lang in languages:
-                if 'counter' not in languages[lang]:
-                    languages[lang]['counter'] = [engine_name]
-                elif engine_name not in languages[lang]['counter']:
-                    languages[lang]['counter'].append(engine_name)
-
-    # filter list to include only languages supported by most engines
-    min_supported_engines = int(0.70 * len(engines_languages))
-    languages = {code: lang for code, lang
-                 in languages.items()
-                 if len(lang.get('counter', [])) >= min_supported_engines or
-                 len(languages.get(code.split('-')[0], {}).get('counter', [])) >= min_supported_engines}
-
-    # get locales that have no name or country yet
-    for locale in languages.keys():
-        # try to get language names
-        if not languages[locale].get('name'):
-            name = languages.get(locale.split('-')[0], {}).get('name', None)
-            if name:
-                languages[locale]['name'] = name
-            else:
-                # filter out locales with no name
-                del languages[locale]
-                continue
-
-        # try to get language name in english
-        if not languages[locale].get('english_name'):
-            languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '')
-
-        # try to get country name
-        if locale.find('-') > 0 and not languages[locale].get('country'):
-            languages[locale]['country'] = get_country_name(locale) or ''
-
-
-# Remove countryless language if language is featured in only one country.
-def filter_single_country_languages():
-    prev_lang = None
-    prev_code = None
-    for code in sorted(languages):
-        lang = code.split('-')[0]
-        if lang == prev_lang:
+        for lang_code in engines_languages[engine_name]:
+
+            # apply custom fixes if necessary
+            if lang_code.lower() in locale_fixes:
+                lang_code = locale_fixes[lang_code.lower()]
+
+            locale = get_locale(lang_code)
+
+            # ensure that lang_code uses standard language and country codes
+            if locale and locale.territory:
+                lang_code = locale.language + '-' + locale.territory
+
+            # add locale if it's not in list
+            if lang_code not in language_list:
+                if locale:
+                    language_list[lang_code] = {'name': locale.get_language_name().title(),
+                                                'english_name': locale.english_name,
+                                                'country': locale.get_territory_name() or ''}
+
+                    # also add language without country
+                    if locale.language not in language_list:
+                        language_list[locale.language] = {'name': locale.get_language_name().title(),
+                                                          'english_name': locale.english_name}
+                else:
+                    language_list[lang_code] = {}
+
+            # count engine for both language_country combination and language alone
+            add_engine_counter(lang_code, engine_name, language_list)
+            add_engine_counter(lang_code.split('-')[0], engine_name, language_list)
+
+    return language_list
+
+
+# Filter language list so it only includes the most supported languages and countries.
+def filter_language_list(all_languages):
+    min_supported_engines = 10
+    main_engines = [engine_name for engine_name in engines.keys()
+                    if 'general' in engines[engine_name].categories and
+                       engines[engine_name].supported_languages and
+                       not engines[engine_name].disabled]
+
+    # filter list to include only languages supported by most engines or all default general engines
+    filtered_languages = {code: lang for code, lang
+                          in all_languages.items()
+                          if (len(lang.get('counter', [])) >= min_supported_engines or
+                              all(main_engine in lang.get('counter', [])
+                                  for main_engine in main_engines))}
+
+    return filtered_languages
+
+
+# Add country codes to languages without one and filter out language codes.
+def assign_country_codes(filtered_languages, all_languages):
+    sorted_languages = sorted(all_languages,
+                              key=lambda lang: len(all_languages[lang].get('counter', [])),
+                              reverse=True)
+    previous_lang = None
+    previous_code = None
+    countries = 0
+    for current_code in sorted(filtered_languages):
+        current_lang = current_code.split('-')[0]
+
+        # count country codes per language
+        if current_lang == previous_lang:
             countries += 1
+
         else:
-            if prev_lang is not None and countries == 1:
-                del languages[prev_lang]
-                languages[prev_code]['country'] = ''
+            if previous_lang is not None:
+                # if language has no single country code
+                if countries == 0:
+                    # try to get country code with most supported engines
+                    for l in sorted_languages:
+                        l_parts = l.split('-')
+                        if len(l_parts) == 2 and l_parts[0] == previous_lang:
+                            filtered_languages[l] = all_languages[l]
+                            filtered_languages[l]['country'] = ''
+                            countries = 1
+                            break
+
+                    if countries == 0:
+                        # get most likely country code from babel
+                        subtags = get_global('likely_subtags').get(previous_lang)
+                        if subtags:
+                            subtag_parts = subtags.split('_')
+                            new_code = subtag_parts[0] + '-' + subtag_parts[-1]
+                            filtered_languages[new_code] = all_languages[previous_lang]
+                            countries = 1
+
+                if countries == 1:
+                    # remove countryless version of language if there's only one country
+                    del filtered_languages[previous_lang]
+                    if previous_code in filtered_languages:
+                        filtered_languages[previous_code]['country'] = ''
+
             countries = 0
-            prev_lang = lang
-        prev_code = code
+            previous_lang = current_lang
+
+        previous_code = current_code
 
 
 # Write languages.py.
-def write_languages_file():
+def write_languages_file(languages):
     new_file = open(languages_file, 'wb')
     file_content = '# -*- coding: utf-8 -*-\n'\
                    + '# list of language codes\n'\
@@ -183,7 +193,9 @@ def write_languages_file():
 
 
 if __name__ == "__main__":
-    fetch_supported_languages()
-    join_language_lists()
-    filter_single_country_languages()
-    write_languages_file()
+    initialize_engines(settings['engines'])
+    engines_languages = fetch_supported_languages()
+    all_languages = join_language_lists(engines_languages)
+    filtered_languages = filter_language_list(all_languages)
+    assign_country_codes(filtered_languages, all_languages)
+    write_languages_file(filtered_languages)