[mod] bing: add time_range support & upgrade to data_type: traits_v1

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/data/engine_traits.json

@@ -2,2018 +2,366 @@
   "bing": {
     "all_locale": null,
     "custom": {},
-    "data_type": "supported_languages",
+    "data_type": "traits_v1",
     "languages": {
-      "af": "af",
-      "am": "am",
       "ar": "ar",
-      "as": "as",
-      "az": "az",
-      "be": "be",
       "bg": "bg",
       "bn": "bn",
-      "bs": "bs",
       "ca": "ca",
-      "chr": "chr",
       "cs": "cs",
-      "cy": "cy",
       "da": "da",
       "de": "de",
-      "el": "el",
       "en": "en",
       "es": "es",
       "et": "et",
       "eu": "eu",
-      "fa": "fa",
       "fi": "fi",
-      "fil": "fil",
       "fr": "fr",
-      "ga": "ga",
-      "gd": "gd",
       "gl": "gl",
       "gu": "gu",
-      "ha": "ha",
       "he": "he",
       "hi": "hi",
       "hr": "hr",
       "hu": "hu",
-      "hy": "hy",
-      "id": "id",
-      "ig": "ig",
       "is": "is",
       "it": "it",
-      "ja": "ja",
-      "ka": "ka",
-      "kk": "kk",
-      "km": "km",
+      "ja": "jp",
       "kn": "kn",
       "ko": "ko",
-      "kok": "kok",
-      "ku": "ku",
-      "ky": "ky",
-      "lb": "lb",
-      "lo": "lo",
       "lt": "lt",
       "lv": "lv",
-      "mi": "mi",
-      "mk": "mk",
       "ml": "ml",
-      "mn": "MN",
       "mr": "mr",
       "ms": "ms",
-      "mt": "mt",
       "nb": "nb",
-      "ne": "ne",
       "nl": "nl",
-      "nn": "nn",
-      "or": "or",
       "pa": "pa",
-      "pa_Arab": "pa-Arab",
       "pl": "pl",
-      "pt": "pt",
-      "qu": "quz",
+      "pt": "pt-pt",
       "ro": "ro",
       "ru": "ru",
-      "rw": "rw",
-      "sd": "sd",
-      "si": "si",
       "sk": "sk",
       "sl": "sl",
-      "sq": "sq",
       "sr": "sr",
-      "sr_Latn": "sr_latn",
       "sv": "sv",
-      "sw": "sw",
       "ta": "ta",
       "te": "te",
-      "tg": "tg",
       "th": "th",
-      "ti": "ti",
-      "tk": "tk",
       "tr": "tr",
-      "tt": "tt",
-      "ug": "ug",
       "uk": "uk",
-      "ur": "ur",
-      "uz": "uz",
       "vi": "vi",
-      "wo": "wo",
-      "xh": "xh",
-      "yo": "yo",
-      "zh": "zh_Hans",
-      "zh_Hans": "zh_Hans",
-      "zh_Hant": "zh_Hant",
-      "zu": "zu"
+      "zh": "zh-hans",
+      "zh_Hans": "zh-hans",
+      "zh_Hant": "zh-hant"
     },
     "regions": {
-      "am-ET": "et",
-      "ar-AE": "ae",
-      "ar-BH": "bh",
-      "ar-DJ": "dj",
-      "ar-DZ": "dz",
-      "ar-EG": "eg",
-      "ar-ER": "er",
-      "ar-IL": "il",
-      "ar-IQ": "iq",
-      "ar-JO": "jo",
-      "ar-KM": "km",
-      "ar-KW": "kw",
-      "ar-LB": "lb",
-      "ar-LY": "ly",
-      "ar-MA": "ma",
-      "ar-MR": "mr",
-      "ar-OM": "om",
-      "ar-PS": "ps",
-      "ar-QA": "qa",
-      "ar-SA": "sa",
-      "ar-SD": "sd",
-      "ar-SO": "so",
-      "ar-SY": "sy",
-      "ar-TD": "td",
-      "ar-TN": "tn",
-      "ar-YE": "ye",
-      "az-AZ": "az",
-      "be-BY": "by",
-      "bg-BG": "bg",
-      "bn-BD": "bd",
-      "bs-BA": "ba",
-      "ca-AD": "ad",
-      "cs-CZ": "cz",
-      "da-DK": "dk",
-      "de-AT": "at",
-      "de-BE": "be",
-      "de-CH": "ch",
-      "de-DE": "de",
-      "de-LI": "li",
-      "de-LU": "lu",
-      "el-CY": "cy",
-      "el-GR": "gr",
-      "en-AG": "ag",
-      "en-AI": "ai",
-      "en-AS": "as",
-      "en-AU": "au",
-      "en-BB": "bb",
-      "en-BI": "bi",
-      "en-BM": "bm",
-      "en-BS": "bs",
-      "en-BW": "bw",
-      "en-BZ": "bz",
-      "en-CA": "ca",
-      "en-CC": "cc",
-      "en-CK": "ck",
-      "en-CM": "cm",
-      "en-CX": "cx",
-      "en-DM": "dm",
-      "en-ER": "er",
-      "en-FJ": "fj",
-      "en-FK": "fk",
-      "en-FM": "fm",
-      "en-GB": "gb",
-      "en-GD": "gd",
-      "en-GG": "gg",
-      "en-GH": "gh",
-      "en-GI": "gi",
-      "en-GM": "gm",
-      "en-GU": "gu",
-      "en-GY": "gy",
-      "en-HK": "hk",
-      "en-IE": "ie",
-      "en-IN": "in",
-      "en-JE": "je",
-      "en-JM": "jm",
-      "en-KE": "ke",
-      "en-KI": "ki",
-      "en-KN": "kn",
-      "en-KY": "ky",
-      "en-LC": "lc",
-      "en-LR": "lr",
-      "en-LS": "ls",
-      "en-MG": "mg",
-      "en-MH": "mh",
-      "en-MP": "mp",
-      "en-MS": "ms",
-      "en-MT": "mt",
-      "en-MU": "mu",
-      "en-MW": "mw",
-      "en-NA": "na",
-      "en-NF": "nf",
-      "en-NG": "ng",
-      "en-NR": "nr",
-      "en-NU": "nu",
-      "en-NZ": "nz",
-      "en-PG": "pg",
-      "en-PH": "ph",
-      "en-PK": "pk",
-      "en-PN": "pn",
-      "en-PR": "pr",
-      "en-PW": "pw",
-      "en-RW": "rw",
-      "en-SB": "sb",
-      "en-SC": "sc",
-      "en-SD": "sd",
-      "en-SG": "sg",
-      "en-SH": "sh",
-      "en-SL": "sl",
-      "en-SS": "ss",
-      "en-SX": "sx",
-      "en-SZ": "sz",
-      "en-TC": "tc",
-      "en-TK": "tk",
-      "en-TO": "to",
-      "en-TT": "tt",
-      "en-TV": "tv",
-      "en-TZ": "tz",
-      "en-UG": "ug",
-      "en-US": "us",
-      "en-VC": "vc",
-      "en-VG": "vg",
-      "en-VI": "vi",
-      "en-VU": "vu",
-      "en-WS": "ws",
-      "en-ZA": "za",
-      "en-ZM": "zm",
-      "en-ZW": "zw",
-      "es-AR": "ar",
-      "es-BO": "bo",
-      "es-CL": "cl",
-      "es-CO": "co",
-      "es-CR": "cr",
-      "es-DO": "do",
-      "es-EC": "ec",
-      "es-ES": "es",
-      "es-GQ": "gq",
-      "es-GT": "gt",
-      "es-HN": "hn",
-      "es-MX": "mx",
-      "es-NI": "ni",
-      "es-PA": "pa",
-      "es-PE": "pe",
-      "es-PR": "pr",
-      "es-PY": "py",
-      "es-SV": "sv",
-      "es-UY": "uy",
-      "es-VE": "ve",
-      "et-EE": "ee",
-      "fa-AF": "af",
-      "fa-IR": "ir",
-      "fi-FI": "fi",
-      "fil-PH": "ph",
-      "fr-BE": "be",
-      "fr-BF": "bf",
-      "fr-BI": "bi",
-      "fr-BJ": "bj",
-      "fr-BL": "bl",
-      "fr-CA": "ca",
-      "fr-CD": "cd",
-      "fr-CF": "cf",
-      "fr-CG": "cg",
-      "fr-CH": "ch",
-      "fr-CI": "ci",
-      "fr-CM": "cm",
-      "fr-DJ": "dj",
-      "fr-DZ": "dz",
-      "fr-FR": "fr",
-      "fr-GA": "ga",
-      "fr-GF": "gf",
-      "fr-GN": "gn",
-      "fr-GP": "gp",
-      "fr-GQ": "gq",
-      "fr-HT": "ht",
-      "fr-KM": "km",
-      "fr-LU": "lu",
-      "fr-MA": "ma",
-      "fr-MC": "mc",
-      "fr-MF": "mf",
-      "fr-MG": "mg",
-      "fr-ML": "ml",
-      "fr-MQ": "mq",
-      "fr-MU": "mu",
-      "fr-NC": "nc",
-      "fr-NE": "ne",
-      "fr-PF": "pf",
-      "fr-PM": "pm",
-      "fr-RE": "re",
-      "fr-RW": "rw",
-      "fr-SC": "sc",
-      "fr-SN": "sn",
-      "fr-SY": "sy",
-      "fr-TD": "td",
-      "fr-TG": "tg",
-      "fr-TN": "tn",
-      "fr-VU": "vu",
-      "fr-WF": "wf",
-      "fr-YT": "yt",
-      "ga-IE": "ie",
-      "he-IL": "il",
-      "hi-IN": "in",
-      "hr-BA": "ba",
-      "hr-HR": "hr",
-      "hu-HU": "hu",
-      "hy-AM": "am",
-      "id-ID": "id",
-      "is-IS": "is",
-      "it-CH": "ch",
-      "it-IT": "it",
-      "it-SM": "sm",
-      "it-VA": "va",
-      "ja-JP": "jp",
-      "ka-GE": "ge",
-      "kk-KZ": "kz",
-      "km-KH": "kh",
-      "ko-KR": "kr",
-      "ky-KG": "kg",
-      "lb-LU": "lu",
-      "lo-LA": "la",
-      "lt-LT": "lt",
-      "lv-LV": "lv",
-      "mi-NZ": "nz",
-      "mk-MK": "mk",
-      "mn-MN": "mn",
-      "ms-BN": "bn",
-      "ms-MY": "my",
-      "ms-SG": "sg",
-      "mt-MT": "mt",
-      "nb-NO": "no",
-      "ne-NP": "np",
-      "nl-AW": "aw",
-      "nl-BE": "be",
-      "nl-CW": "cw",
-      "nl-NL": "nl",
-      "nl-SR": "sr",
-      "nl-SX": "sx",
-      "nn-NO": "no",
-      "pl-PL": "pl",
-      "pt-AO": "ao",
-      "pt-BR": "br",
-      "pt-CV": "cv",
-      "pt-GQ": "gq",
-      "pt-GW": "gw",
-      "pt-MO": "mo",
-      "pt-MZ": "mz",
-      "pt-PT": "pt",
-      "pt-ST": "st",
-      "qu-BO": "bo",
-      "qu-EC": "ec",
-      "qu-PE": "pe",
-      "ro-MD": "md",
-      "ro-RO": "ro",
-      "ru-BY": "by",
-      "ru-KG": "kg",
-      "ru-KZ": "kz",
-      "ru-RU": "ru",
-      "ru-UA": "ua",
-      "rw-RW": "rw",
-      "si-LK": "lk",
-      "sk-SK": "sk",
-      "sl-SI": "si",
-      "sq-AL": "al",
-      "sr-BA": "ba",
-      "sr-ME": "me",
-      "sr-RS": "rs",
-      "sv-FI": "fi",
-      "sv-SE": "se",
-      "sw-KE": "ke",
-      "sw-TZ": "tz",
-      "sw-UG": "ug",
-      "ta-LK": "lk",
-      "ta-SG": "sg",
-      "tg-TJ": "tj",
-      "th-TH": "th",
-      "ti-ER": "er",
-      "tk-TM": "tm",
-      "tr-CY": "cy",
-      "tr-TR": "tr",
-      "uk-UA": "ua",
-      "ur-PK": "pk",
-      "uz-UZ": "uz",
-      "vi-VN": "vn",
-      "wo-SN": "sn",
-      "yo-NG": "ng",
-      "zh-CN": "cn",
-      "zh-HK": "hk",
-      "zh-MO": "mo",
-      "zh-SG": "sg",
-      "zh-TW": "tw"
+      "da-DK": "da-DK",
+      "de-AT": "de-AT",
+      "de-CH": "de-CH",
+      "de-DE": "de-DE",
+      "en-AU": "en-AU",
+      "en-CA": "en-CA",
+      "en-GB": "en-GB",
+      "en-IN": "en-IN",
+      "en-MY": "en-MY",
+      "en-NZ": "en-NZ",
+      "en-PH": "en-PH",
+      "en-US": "en-US",
+      "en-ZA": "en-ZA",
+      "es-AR": "es-AR",
+      "es-CL": "es-CL",
+      "es-ES": "es-ES",
+      "es-MX": "es-MX",
+      "es-US": "es-US",
+      "fi-FI": "fi-FI",
+      "fr-BE": "fr-BE",
+      "fr-CA": "fr-CA",
+      "fr-CH": "fr-CH",
+      "fr-FR": "fr-FR",
+      "id-ID": "en-ID",
+      "it-IT": "it-IT",
+      "ja-JP": "ja-JP",
+      "ko-KR": "ko-KR",
+      "nb-NO": "no-NO",
+      "nl-BE": "nl-BE",
+      "nl-NL": "nl-NL",
+      "pl-PL": "pl-PL",
+      "pt-BR": "pt-BR",
+      "ru-RU": "ru-RU",
+      "sv-SE": "sv-SE",
+      "tr-TR": "tr-TR",
+      "zh-CN": "zh-CN",
+      "zh-HK": "zh-HK",
+      "zh-TW": "zh-TW"
     },
-    "supported_languages": [
-      "af",
-      "am",
-      "ar",
-      "as",
-      "az-latn",
-      "be",
-      "bg",
-      "bn",
-      "bs-latn",
-      "ca",
-      "ca-es-valencia",
-      "chr-cher",
-      "cs",
-      "cy",
-      "da",
-      "de",
-      "el",
-      "en",
-      "es",
-      "et",
-      "eu",
-      "fa",
-      "fi",
-      "fil",
-      "fr",
-      "ga",
-      "gd",
-      "gl",
-      "gu",
-      "ha-latn",
-      "he",
-      "hi",
-      "hr",
-      "hu",
-      "hy",
-      "id",
-      "ig",
-      "is",
-      "it",
-      "ja",
-      "ka",
-      "kk",
-      "km",
-      "kn",
-      "ko",
-      "kok",
-      "ku-arab",
-      "ky",
-      "lb",
-      "lo",
-      "lt",
-      "lv",
-      "mi",
-      "mk",
-      "ml",
-      "mn-Cyrl-MN",
-      "mr",
-      "ms",
-      "mt",
-      "nb",
-      "ne",
-      "nl",
-      "nn",
-      "nso",
-      "or",
-      "pa-arab",
-      "pa-guru",
-      "pl",
-      "prs",
-      "pt-BR",
-      "pt-PT",
-      "quc",
-      "quz",
-      "ro",
-      "ru",
-      "rw",
-      "sd-arab",
-      "si",
-      "sk",
-      "sl",
-      "sq",
-      "sr-cyrl",
-      "sr-latn",
-      "sv",
-      "sw",
-      "ta",
-      "te",
-      "tg-cyrl",
-      "th",
-      "ti",
-      "tk",
-      "tn",
-      "tr",
-      "tt",
-      "ug",
-      "uk",
-      "ur",
-      "uz-latn",
-      "vi",
-      "wo",
-      "xh",
-      "yo",
-      "zh-Hans",
-      "zh-Hant",
-      "zu"
-    ]
+    "supported_languages": {}
   },
   "bing images": {
     "all_locale": null,
     "custom": {},
-    "data_type": "supported_languages",
+    "data_type": "traits_v1",
     "languages": {
-      "af": "af",
-      "am": "am",
       "ar": "ar",
-      "as": "as",
-      "az": "az",
-      "be": "be",
       "bg": "bg",
       "bn": "bn",
-      "bs": "bs",
       "ca": "ca",
-      "chr": "chr",
       "cs": "cs",
-      "cy": "cy",
       "da": "da",
       "de": "de",
-      "el": "el",
       "en": "en",
       "es": "es",
       "et": "et",
       "eu": "eu",
-      "fa": "fa",
       "fi": "fi",
-      "fil": "fil",
       "fr": "fr",
-      "ga": "ga",
-      "gd": "gd",
       "gl": "gl",
       "gu": "gu",
-      "ha": "ha",
       "he": "he",
       "hi": "hi",
       "hr": "hr",
       "hu": "hu",
-      "hy": "hy",
-      "id": "id",
-      "ig": "ig",
       "is": "is",
       "it": "it",
-      "ja": "ja",
-      "ka": "ka",
-      "kk": "kk",
-      "km": "km",
+      "ja": "jp",
       "kn": "kn",
       "ko": "ko",
-      "kok": "kok",
-      "ku": "ku",
-      "ky": "ky",
-      "lb": "lb",
-      "lo": "lo",
       "lt": "lt",
       "lv": "lv",
-      "mi": "mi",
-      "mk": "mk",
       "ml": "ml",
-      "mn": "MN",
       "mr": "mr",
       "ms": "ms",
-      "mt": "mt",
       "nb": "nb",
-      "ne": "ne",
       "nl": "nl",
-      "nn": "nn",
-      "or": "or",
       "pa": "pa",
-      "pa_Arab": "pa-Arab",
       "pl": "pl",
-      "pt": "pt",
-      "qu": "quz",
+      "pt": "pt-pt",
       "ro": "ro",
       "ru": "ru",
-      "rw": "rw",
-      "sd": "sd",
-      "si": "si",
       "sk": "sk",
       "sl": "sl",
-      "sq": "sq",
       "sr": "sr",
-      "sr_Latn": "sr_latn",
       "sv": "sv",
-      "sw": "sw",
       "ta": "ta",
       "te": "te",
-      "tg": "tg",
       "th": "th",
-      "ti": "ti",
-      "tk": "tk",
       "tr": "tr",
-      "tt": "tt",
-      "ug": "ug",
       "uk": "uk",
-      "ur": "ur",
-      "uz": "uz",
       "vi": "vi",
-      "wo": "wo",
-      "xh": "xh",
-      "yo": "yo",
-      "zh": "zh_Hans",
-      "zh_Hans": "zh_Hans",
-      "zh_Hant": "zh_Hant",
-      "zu": "zu"
+      "zh": "zh-hans",
+      "zh_Hans": "zh-hans",
+      "zh_Hant": "zh-hant"
     },
     "regions": {
-      "am-ET": "et",
-      "ar-AE": "ae",
-      "ar-BH": "bh",
-      "ar-DJ": "dj",
-      "ar-DZ": "dz",
-      "ar-EG": "eg",
-      "ar-ER": "er",
-      "ar-IL": "il",
-      "ar-IQ": "iq",
-      "ar-JO": "jo",
-      "ar-KM": "km",
-      "ar-KW": "kw",
-      "ar-LB": "lb",
-      "ar-LY": "ly",
-      "ar-MA": "ma",
-      "ar-MR": "mr",
-      "ar-OM": "om",
-      "ar-PS": "ps",
-      "ar-QA": "qa",
-      "ar-SA": "sa",
-      "ar-SD": "sd",
-      "ar-SO": "so",
-      "ar-SY": "sy",
-      "ar-TD": "td",
-      "ar-TN": "tn",
-      "ar-YE": "ye",
-      "az-AZ": "az",
-      "be-BY": "by",
-      "bg-BG": "bg",
-      "bn-BD": "bd",
-      "bs-BA": "ba",
-      "ca-AD": "ad",
-      "cs-CZ": "cz",
-      "da-DK": "dk",
-      "de-AT": "at",
-      "de-BE": "be",
-      "de-CH": "ch",
-      "de-DE": "de",
-      "de-LI": "li",
-      "de-LU": "lu",
-      "el-CY": "cy",
-      "el-GR": "gr",
-      "en-AG": "ag",
-      "en-AI": "ai",
-      "en-AS": "as",
-      "en-AU": "au",
-      "en-BB": "bb",
-      "en-BI": "bi",
-      "en-BM": "bm",
-      "en-BS": "bs",
-      "en-BW": "bw",
-      "en-BZ": "bz",
-      "en-CA": "ca",
-      "en-CC": "cc",
-      "en-CK": "ck",
-      "en-CM": "cm",
-      "en-CX": "cx",
-      "en-DM": "dm",
-      "en-ER": "er",
-      "en-FJ": "fj",
-      "en-FK": "fk",
-      "en-FM": "fm",
-      "en-GB": "gb",
-      "en-GD": "gd",
-      "en-GG": "gg",
-      "en-GH": "gh",
-      "en-GI": "gi",
-      "en-GM": "gm",
-      "en-GU": "gu",
-      "en-GY": "gy",
-      "en-HK": "hk",
-      "en-IE": "ie",
-      "en-IN": "in",
-      "en-JE": "je",
-      "en-JM": "jm",
-      "en-KE": "ke",
-      "en-KI": "ki",
-      "en-KN": "kn",
-      "en-KY": "ky",
-      "en-LC": "lc",
-      "en-LR": "lr",
-      "en-LS": "ls",
-      "en-MG": "mg",
-      "en-MH": "mh",
-      "en-MP": "mp",
-      "en-MS": "ms",
-      "en-MT": "mt",
-      "en-MU": "mu",
-      "en-MW": "mw",
-      "en-NA": "na",
-      "en-NF": "nf",
-      "en-NG": "ng",
-      "en-NR": "nr",
-      "en-NU": "nu",
-      "en-NZ": "nz",
-      "en-PG": "pg",
-      "en-PH": "ph",
-      "en-PK": "pk",
-      "en-PN": "pn",
-      "en-PR": "pr",
-      "en-PW": "pw",
-      "en-RW": "rw",
-      "en-SB": "sb",
-      "en-SC": "sc",
-      "en-SD": "sd",
-      "en-SG": "sg",
-      "en-SH": "sh",
-      "en-SL": "sl",
-      "en-SS": "ss",
-      "en-SX": "sx",
-      "en-SZ": "sz",
-      "en-TC": "tc",
-      "en-TK": "tk",
-      "en-TO": "to",
-      "en-TT": "tt",
-      "en-TV": "tv",
-      "en-TZ": "tz",
-      "en-UG": "ug",
-      "en-US": "us",
-      "en-VC": "vc",
-      "en-VG": "vg",
-      "en-VI": "vi",
-      "en-VU": "vu",
-      "en-WS": "ws",
-      "en-ZA": "za",
-      "en-ZM": "zm",
-      "en-ZW": "zw",
-      "es-AR": "ar",
-      "es-BO": "bo",
-      "es-CL": "cl",
-      "es-CO": "co",
-      "es-CR": "cr",
-      "es-DO": "do",
-      "es-EC": "ec",
-      "es-ES": "es",
-      "es-GQ": "gq",
-      "es-GT": "gt",
-      "es-HN": "hn",
-      "es-MX": "mx",
-      "es-NI": "ni",
-      "es-PA": "pa",
-      "es-PE": "pe",
-      "es-PR": "pr",
-      "es-PY": "py",
-      "es-SV": "sv",
-      "es-UY": "uy",
-      "es-VE": "ve",
-      "et-EE": "ee",
-      "fa-AF": "af",
-      "fa-IR": "ir",
-      "fi-FI": "fi",
-      "fil-PH": "ph",
-      "fr-BE": "be",
-      "fr-BF": "bf",
-      "fr-BI": "bi",
-      "fr-BJ": "bj",
-      "fr-BL": "bl",
-      "fr-CA": "ca",
-      "fr-CD": "cd",
-      "fr-CF": "cf",
-      "fr-CG": "cg",
-      "fr-CH": "ch",
-      "fr-CI": "ci",
-      "fr-CM": "cm",
-      "fr-DJ": "dj",
-      "fr-DZ": "dz",
-      "fr-FR": "fr",
-      "fr-GA": "ga",
-      "fr-GF": "gf",
-      "fr-GN": "gn",
-      "fr-GP": "gp",
-      "fr-GQ": "gq",
-      "fr-HT": "ht",
-      "fr-KM": "km",
-      "fr-LU": "lu",
-      "fr-MA": "ma",
-      "fr-MC": "mc",
-      "fr-MF": "mf",
-      "fr-MG": "mg",
-      "fr-ML": "ml",
-      "fr-MQ": "mq",
-      "fr-MU": "mu",
-      "fr-NC": "nc",
-      "fr-NE": "ne",
-      "fr-PF": "pf",
-      "fr-PM": "pm",
-      "fr-RE": "re",
-      "fr-RW": "rw",
-      "fr-SC": "sc",
-      "fr-SN": "sn",
-      "fr-SY": "sy",
-      "fr-TD": "td",
-      "fr-TG": "tg",
-      "fr-TN": "tn",
-      "fr-VU": "vu",
-      "fr-WF": "wf",
-      "fr-YT": "yt",
-      "ga-IE": "ie",
-      "he-IL": "il",
-      "hi-IN": "in",
-      "hr-BA": "ba",
-      "hr-HR": "hr",
-      "hu-HU": "hu",
-      "hy-AM": "am",
-      "id-ID": "id",
-      "is-IS": "is",
-      "it-CH": "ch",
-      "it-IT": "it",
-      "it-SM": "sm",
-      "it-VA": "va",
-      "ja-JP": "jp",
-      "ka-GE": "ge",
-      "kk-KZ": "kz",
-      "km-KH": "kh",
-      "ko-KR": "kr",
-      "ky-KG": "kg",
-      "lb-LU": "lu",
-      "lo-LA": "la",
-      "lt-LT": "lt",
-      "lv-LV": "lv",
-      "mi-NZ": "nz",
-      "mk-MK": "mk",
-      "mn-MN": "mn",
-      "ms-BN": "bn",
-      "ms-MY": "my",
-      "ms-SG": "sg",
-      "mt-MT": "mt",
-      "nb-NO": "no",
-      "ne-NP": "np",
-      "nl-AW": "aw",
-      "nl-BE": "be",
-      "nl-CW": "cw",
-      "nl-NL": "nl",
-      "nl-SR": "sr",
-      "nl-SX": "sx",
-      "nn-NO": "no",
-      "pl-PL": "pl",
-      "pt-AO": "ao",
-      "pt-BR": "br",
-      "pt-CV": "cv",
-      "pt-GQ": "gq",
-      "pt-GW": "gw",
-      "pt-MO": "mo",
-      "pt-MZ": "mz",
-      "pt-PT": "pt",
-      "pt-ST": "st",
-      "qu-BO": "bo",
-      "qu-EC": "ec",
-      "qu-PE": "pe",
-      "ro-MD": "md",
-      "ro-RO": "ro",
-      "ru-BY": "by",
-      "ru-KG": "kg",
-      "ru-KZ": "kz",
-      "ru-RU": "ru",
-      "ru-UA": "ua",
-      "rw-RW": "rw",
-      "si-LK": "lk",
-      "sk-SK": "sk",
-      "sl-SI": "si",
-      "sq-AL": "al",
-      "sr-BA": "ba",
-      "sr-ME": "me",
-      "sr-RS": "rs",
-      "sv-FI": "fi",
-      "sv-SE": "se",
-      "sw-KE": "ke",
-      "sw-TZ": "tz",
-      "sw-UG": "ug",
-      "ta-LK": "lk",
-      "ta-SG": "sg",
-      "tg-TJ": "tj",
-      "th-TH": "th",
-      "ti-ER": "er",
-      "tk-TM": "tm",
-      "tr-CY": "cy",
-      "tr-TR": "tr",
-      "uk-UA": "ua",
-      "ur-PK": "pk",
-      "uz-UZ": "uz",
-      "vi-VN": "vn",
-      "wo-SN": "sn",
-      "yo-NG": "ng",
-      "zh-CN": "cn",
-      "zh-HK": "hk",
-      "zh-MO": "mo",
-      "zh-SG": "sg",
-      "zh-TW": "tw"
+      "da-DK": "da-DK",
+      "de-AT": "de-AT",
+      "de-CH": "de-CH",
+      "de-DE": "de-DE",
+      "en-AU": "en-AU",
+      "en-CA": "en-CA",
+      "en-GB": "en-GB",
+      "en-IN": "en-IN",
+      "en-MY": "en-MY",
+      "en-NZ": "en-NZ",
+      "en-PH": "en-PH",
+      "en-US": "en-US",
+      "en-ZA": "en-ZA",
+      "es-AR": "es-AR",
+      "es-CL": "es-CL",
+      "es-ES": "es-ES",
+      "es-MX": "es-MX",
+      "es-US": "es-US",
+      "fi-FI": "fi-FI",
+      "fr-BE": "fr-BE",
+      "fr-CA": "fr-CA",
+      "fr-CH": "fr-CH",
+      "fr-FR": "fr-FR",
+      "id-ID": "en-ID",
+      "it-IT": "it-IT",
+      "ja-JP": "ja-JP",
+      "ko-KR": "ko-KR",
+      "nb-NO": "no-NO",
+      "nl-BE": "nl-BE",
+      "nl-NL": "nl-NL",
+      "pl-PL": "pl-PL",
+      "pt-BR": "pt-BR",
+      "ru-RU": "ru-RU",
+      "sv-SE": "sv-SE",
+      "tr-TR": "tr-TR",
+      "zh-CN": "zh-CN",
+      "zh-HK": "zh-HK",
+      "zh-TW": "zh-TW"
     },
-    "supported_languages": [
-      "af",
-      "am",
-      "ar",
-      "as",
-      "az-latn",
-      "be",
-      "bg",
-      "bn",
-      "bs-latn",
-      "ca",
-      "ca-es-valencia",
-      "chr-cher",
-      "cs",
-      "cy",
-      "da",
-      "de",
-      "el",
-      "en",
-      "es",
-      "et",
-      "eu",
-      "fa",
-      "fi",
-      "fil",
-      "fr",
-      "ga",
-      "gd",
-      "gl",
-      "gu",
-      "ha-latn",
-      "he",
-      "hi",
-      "hr",
-      "hu",
-      "hy",
-      "id",
-      "ig",
-      "is",
-      "it",
-      "ja",
-      "ka",
-      "kk",
-      "km",
-      "kn",
-      "ko",
-      "kok",
-      "ku-arab",
-      "ky",
-      "lb",
-      "lo",
-      "lt",
-      "lv",
-      "mi",
-      "mk",
-      "ml",
-      "mn-Cyrl-MN",
-      "mr",
-      "ms",
-      "mt",
-      "nb",
-      "ne",
-      "nl",
-      "nn",
-      "nso",
-      "or",
-      "pa-arab",
-      "pa-guru",
-      "pl",
-      "prs",
-      "pt-BR",
-      "pt-PT",
-      "quc",
-      "quz",
-      "ro",
-      "ru",
-      "rw",
-      "sd-arab",
-      "si",
-      "sk",
-      "sl",
-      "sq",
-      "sr-cyrl",
-      "sr-latn",
-      "sv",
-      "sw",
-      "ta",
-      "te",
-      "tg-cyrl",
-      "th",
-      "ti",
-      "tk",
-      "tn",
-      "tr",
-      "tt",
-      "ug",
-      "uk",
-      "ur",
-      "uz-latn",
-      "vi",
-      "wo",
-      "xh",
-      "yo",
-      "zh-Hans",
-      "zh-Hant",
-      "zu"
-    ]
+    "supported_languages": {}
   },
   "bing news": {
-    "all_locale": null,
+    "all_locale": "en-WW",
     "custom": {},
-    "data_type": "supported_languages",
+    "data_type": "traits_v1",
     "languages": {
-      "af": "af",
-      "am": "am",
       "ar": "ar",
-      "as": "as",
-      "az": "az",
-      "be": "be",
       "bg": "bg",
       "bn": "bn",
-      "bs": "bs",
       "ca": "ca",
-      "chr": "chr",
       "cs": "cs",
-      "cy": "cy",
       "da": "da",
       "de": "de",
-      "el": "el",
       "en": "en",
       "es": "es",
       "et": "et",
       "eu": "eu",
-      "fa": "fa",
       "fi": "fi",
-      "fil": "fil",
       "fr": "fr",
-      "ga": "ga",
-      "gd": "gd",
       "gl": "gl",
       "gu": "gu",
-      "ha": "ha",
       "he": "he",
       "hi": "hi",
       "hr": "hr",
       "hu": "hu",
-      "hy": "hy",
-      "id": "id",
-      "ig": "ig",
       "is": "is",
       "it": "it",
-      "ja": "ja",
-      "ka": "ka",
-      "kk": "kk",
-      "km": "km",
+      "ja": "jp",
       "kn": "kn",
       "ko": "ko",
-      "kok": "kok",
-      "ku": "ku",
-      "ky": "ky",
-      "lb": "lb",
-      "lo": "lo",
       "lt": "lt",
       "lv": "lv",
-      "mi": "mi",
-      "mk": "mk",
       "ml": "ml",
-      "mn": "MN",
       "mr": "mr",
       "ms": "ms",
-      "mt": "mt",
       "nb": "nb",
-      "ne": "ne",
       "nl": "nl",
-      "nn": "nn",
-      "or": "or",
       "pa": "pa",
-      "pa_Arab": "pa-Arab",
       "pl": "pl",
-      "pt": "pt",
-      "qu": "quz",
+      "pt": "pt-pt",
       "ro": "ro",
       "ru": "ru",
-      "rw": "rw",
-      "sd": "sd",
-      "si": "si",
       "sk": "sk",
       "sl": "sl",
-      "sq": "sq",
       "sr": "sr",
-      "sr_Latn": "sr_latn",
       "sv": "sv",
-      "sw": "sw",
       "ta": "ta",
       "te": "te",
-      "tg": "tg",
       "th": "th",
-      "ti": "ti",
-      "tk": "tk",
       "tr": "tr",
-      "tt": "tt",
-      "ug": "ug",
       "uk": "uk",
-      "ur": "ur",
-      "uz": "uz",
       "vi": "vi",
-      "wo": "wo",
-      "xh": "xh",
-      "yo": "yo",
-      "zh": "zh_Hans",
-      "zh_Hans": "zh_Hans",
-      "zh_Hant": "zh_Hant",
-      "zu": "zu"
+      "zh": "zh-hans",
+      "zh_Hans": "zh-hans",
+      "zh_Hant": "zh-hant"
     },
     "regions": {
-      "am-ET": "et",
-      "ar-AE": "ae",
-      "ar-BH": "bh",
-      "ar-DJ": "dj",
-      "ar-DZ": "dz",
-      "ar-EG": "eg",
-      "ar-ER": "er",
-      "ar-IL": "il",
-      "ar-IQ": "iq",
-      "ar-JO": "jo",
-      "ar-KM": "km",
-      "ar-KW": "kw",
-      "ar-LB": "lb",
-      "ar-LY": "ly",
-      "ar-MA": "ma",
-      "ar-MR": "mr",
-      "ar-OM": "om",
-      "ar-PS": "ps",
-      "ar-QA": "qa",
-      "ar-SA": "sa",
-      "ar-SD": "sd",
-      "ar-SO": "so",
-      "ar-SY": "sy",
-      "ar-TD": "td",
-      "ar-TN": "tn",
-      "ar-YE": "ye",
-      "az-AZ": "az",
-      "be-BY": "by",
-      "bg-BG": "bg",
-      "bn-BD": "bd",
-      "bs-BA": "ba",
-      "ca-AD": "ad",
-      "cs-CZ": "cz",
-      "da-DK": "dk",
-      "de-AT": "at",
-      "de-BE": "be",
-      "de-CH": "ch",
-      "de-DE": "de",
-      "de-LI": "li",
-      "de-LU": "lu",
-      "el-CY": "cy",
-      "el-GR": "gr",
-      "en-AG": "ag",
-      "en-AI": "ai",
-      "en-AS": "as",
-      "en-AU": "au",
-      "en-BB": "bb",
-      "en-BI": "bi",
-      "en-BM": "bm",
-      "en-BS": "bs",
-      "en-BW": "bw",
-      "en-BZ": "bz",
-      "en-CA": "ca",
-      "en-CC": "cc",
-      "en-CK": "ck",
-      "en-CM": "cm",
-      "en-CX": "cx",
-      "en-DM": "dm",
-      "en-ER": "er",
-      "en-FJ": "fj",
-      "en-FK": "fk",
-      "en-FM": "fm",
-      "en-GB": "gb",
-      "en-GD": "gd",
-      "en-GG": "gg",
-      "en-GH": "gh",
-      "en-GI": "gi",
-      "en-GM": "gm",
-      "en-GU": "gu",
-      "en-GY": "gy",
-      "en-HK": "hk",
-      "en-IE": "ie",
-      "en-IN": "in",
-      "en-JE": "je",
-      "en-JM": "jm",
-      "en-KE": "ke",
-      "en-KI": "ki",
-      "en-KN": "kn",
-      "en-KY": "ky",
-      "en-LC": "lc",
-      "en-LR": "lr",
-      "en-LS": "ls",
-      "en-MG": "mg",
-      "en-MH": "mh",
-      "en-MP": "mp",
-      "en-MS": "ms",
-      "en-MT": "mt",
-      "en-MU": "mu",
-      "en-MW": "mw",
-      "en-NA": "na",
-      "en-NF": "nf",
-      "en-NG": "ng",
-      "en-NR": "nr",
-      "en-NU": "nu",
-      "en-NZ": "nz",
-      "en-PG": "pg",
-      "en-PH": "ph",
-      "en-PK": "pk",
-      "en-PN": "pn",
-      "en-PR": "pr",
-      "en-PW": "pw",
-      "en-RW": "rw",
-      "en-SB": "sb",
-      "en-SC": "sc",
-      "en-SD": "sd",
-      "en-SG": "sg",
-      "en-SH": "sh",
-      "en-SL": "sl",
-      "en-SS": "ss",
-      "en-SX": "sx",
-      "en-SZ": "sz",
-      "en-TC": "tc",
-      "en-TK": "tk",
-      "en-TO": "to",
-      "en-TT": "tt",
-      "en-TV": "tv",
-      "en-TZ": "tz",
-      "en-UG": "ug",
-      "en-US": "us",
-      "en-VC": "vc",
-      "en-VG": "vg",
-      "en-VI": "vi",
-      "en-VU": "vu",
-      "en-WS": "ws",
-      "en-ZA": "za",
-      "en-ZM": "zm",
-      "en-ZW": "zw",
-      "es-AR": "ar",
-      "es-BO": "bo",
-      "es-CL": "cl",
-      "es-CO": "co",
-      "es-CR": "cr",
-      "es-DO": "do",
-      "es-EC": "ec",
-      "es-ES": "es",
-      "es-GQ": "gq",
-      "es-GT": "gt",
-      "es-HN": "hn",
-      "es-MX": "mx",
-      "es-NI": "ni",
-      "es-PA": "pa",
-      "es-PE": "pe",
-      "es-PR": "pr",
-      "es-PY": "py",
-      "es-SV": "sv",
-      "es-UY": "uy",
-      "es-VE": "ve",
-      "et-EE": "ee",
-      "fa-AF": "af",
-      "fa-IR": "ir",
-      "fi-FI": "fi",
-      "fil-PH": "ph",
-      "fr-BE": "be",
-      "fr-BF": "bf",
-      "fr-BI": "bi",
-      "fr-BJ": "bj",
-      "fr-BL": "bl",
-      "fr-CA": "ca",
-      "fr-CD": "cd",
-      "fr-CF": "cf",
-      "fr-CG": "cg",
-      "fr-CH": "ch",
-      "fr-CI": "ci",
-      "fr-CM": "cm",
-      "fr-DJ": "dj",
-      "fr-DZ": "dz",
-      "fr-FR": "fr",
-      "fr-GA": "ga",
-      "fr-GF": "gf",
-      "fr-GN": "gn",
-      "fr-GP": "gp",
-      "fr-GQ": "gq",
-      "fr-HT": "ht",
-      "fr-KM": "km",
-      "fr-LU": "lu",
-      "fr-MA": "ma",
-      "fr-MC": "mc",
-      "fr-MF": "mf",
-      "fr-MG": "mg",
-      "fr-ML": "ml",
-      "fr-MQ": "mq",
-      "fr-MU": "mu",
-      "fr-NC": "nc",
-      "fr-NE": "ne",
-      "fr-PF": "pf",
-      "fr-PM": "pm",
-      "fr-RE": "re",
-      "fr-RW": "rw",
-      "fr-SC": "sc",
-      "fr-SN": "sn",
-      "fr-SY": "sy",
-      "fr-TD": "td",
-      "fr-TG": "tg",
-      "fr-TN": "tn",
-      "fr-VU": "vu",
-      "fr-WF": "wf",
-      "fr-YT": "yt",
-      "ga-IE": "ie",
-      "he-IL": "il",
-      "hi-IN": "in",
-      "hr-BA": "ba",
-      "hr-HR": "hr",
-      "hu-HU": "hu",
-      "hy-AM": "am",
-      "id-ID": "id",
-      "is-IS": "is",
-      "it-CH": "ch",
-      "it-IT": "it",
-      "it-SM": "sm",
-      "it-VA": "va",
-      "ja-JP": "jp",
-      "ka-GE": "ge",
-      "kk-KZ": "kz",
-      "km-KH": "kh",
-      "ko-KR": "kr",
-      "ky-KG": "kg",
-      "lb-LU": "lu",
-      "lo-LA": "la",
-      "lt-LT": "lt",
-      "lv-LV": "lv",
-      "mi-NZ": "nz",
-      "mk-MK": "mk",
-      "mn-MN": "mn",
-      "ms-BN": "bn",
-      "ms-MY": "my",
-      "ms-SG": "sg",
-      "mt-MT": "mt",
-      "nb-NO": "no",
-      "ne-NP": "np",
-      "nl-AW": "aw",
-      "nl-BE": "be",
-      "nl-CW": "cw",
-      "nl-NL": "nl",
-      "nl-SR": "sr",
-      "nl-SX": "sx",
-      "nn-NO": "no",
-      "pl-PL": "pl",
-      "pt-AO": "ao",
-      "pt-BR": "br",
-      "pt-CV": "cv",
-      "pt-GQ": "gq",
-      "pt-GW": "gw",
-      "pt-MO": "mo",
-      "pt-MZ": "mz",
-      "pt-PT": "pt",
-      "pt-ST": "st",
-      "qu-BO": "bo",
-      "qu-EC": "ec",
-      "qu-PE": "pe",
-      "ro-MD": "md",
-      "ro-RO": "ro",
-      "ru-BY": "by",
-      "ru-KG": "kg",
-      "ru-KZ": "kz",
-      "ru-RU": "ru",
-      "ru-UA": "ua",
-      "rw-RW": "rw",
-      "si-LK": "lk",
-      "sk-SK": "sk",
-      "sl-SI": "si",
-      "sq-AL": "al",
-      "sr-BA": "ba",
-      "sr-ME": "me",
-      "sr-RS": "rs",
-      "sv-FI": "fi",
-      "sv-SE": "se",
-      "sw-KE": "ke",
-      "sw-TZ": "tz",
-      "sw-UG": "ug",
-      "ta-LK": "lk",
-      "ta-SG": "sg",
-      "tg-TJ": "tj",
-      "th-TH": "th",
-      "ti-ER": "er",
-      "tk-TM": "tm",
-      "tr-CY": "cy",
-      "tr-TR": "tr",
-      "uk-UA": "ua",
-      "ur-PK": "pk",
-      "uz-UZ": "uz",
-      "vi-VN": "vn",
-      "wo-SN": "sn",
-      "yo-NG": "ng",
-      "zh-CN": "cn",
-      "zh-HK": "hk",
-      "zh-MO": "mo",
-      "zh-SG": "sg",
-      "zh-TW": "tw"
+      "da-DK": "da-DK",
+      "de-DE": "de-DE",
+      "en-AU": "en-AU",
+      "en-GB": "en-GB",
+      "en-US": "en-US",
+      "es-CL": "es-CL",
+      "es-MX": "es-MX",
+      "es-US": "es-US",
+      "fi-FI": "fi-FI",
+      "fr-CA": "fr-CA",
+      "fr-FR": "fr-FR",
+      "it-IT": "it-IT",
+      "pt-BR": "pt-BR",
+      "zh-CN": "zh-CN"
     },
-    "supported_languages": [
-      "af",
-      "am",
-      "ar",
-      "as",
-      "az-latn",
-      "be",
-      "bg",
-      "bn",
-      "bs-latn",
-      "ca",
-      "ca-es-valencia",
-      "chr-cher",
-      "cs",
-      "cy",
-      "da",
-      "de",
-      "el",
-      "en",
-      "es",
-      "et",
-      "eu",
-      "fa",
-      "fi",
-      "fil",
-      "fr",
-      "ga",
-      "gd",
-      "gl",
-      "gu",
-      "ha-latn",
-      "he",
-      "hi",
-      "hr",
-      "hu",
-      "hy",
-      "id",
-      "ig",
-      "is",
-      "it",
-      "ja",
-      "ka",
-      "kk",
-      "km",
-      "kn",
-      "ko",
-      "kok",
-      "ku-arab",
-      "ky",
-      "lb",
-      "lo",
-      "lt",
-      "lv",
-      "mi",
-      "mk",
-      "ml",
-      "mn-Cyrl-MN",
-      "mr",
-      "ms",
-      "mt",
-      "nb",
-      "ne",
-      "nl",
-      "nn",
-      "nso",
-      "or",
-      "pa-arab",
-      "pa-guru",
-      "pl",
-      "prs",
-      "pt-BR",
-      "pt-PT",
-      "quc",
-      "quz",
-      "ro",
-      "ru",
-      "rw",
-      "sd-arab",
-      "si",
-      "sk",
-      "sl",
-      "sq",
-      "sr-cyrl",
-      "sr-latn",
-      "sv",
-      "sw",
-      "ta",
-      "te",
-      "tg-cyrl",
-      "th",
-      "ti",
-      "tk",
-      "tn",
-      "tr",
-      "tt",
-      "ug",
-      "uk",
-      "ur",
-      "uz-latn",
-      "vi",
-      "wo",
-      "xh",
-      "yo",
-      "zh-Hans",
-      "zh-Hant",
-      "zu"
-    ]
+    "supported_languages": {}
   },
   "bing videos": {
     "all_locale": null,
     "custom": {},
-    "data_type": "supported_languages",
+    "data_type": "traits_v1",
     "languages": {
-      "af": "af",
-      "am": "am",
       "ar": "ar",
-      "as": "as",
-      "az": "az",
-      "be": "be",
       "bg": "bg",
       "bn": "bn",
-      "bs": "bs",
       "ca": "ca",
-      "chr": "chr",
       "cs": "cs",
-      "cy": "cy",
       "da": "da",
       "de": "de",
-      "el": "el",
       "en": "en",
       "es": "es",
       "et": "et",
       "eu": "eu",
-      "fa": "fa",
       "fi": "fi",
-      "fil": "fil",
       "fr": "fr",
-      "ga": "ga",
-      "gd": "gd",
       "gl": "gl",
       "gu": "gu",
-      "ha": "ha",
       "he": "he",
       "hi": "hi",
       "hr": "hr",
       "hu": "hu",
-      "hy": "hy",
-      "id": "id",
-      "ig": "ig",
       "is": "is",
       "it": "it",
-      "ja": "ja",
-      "ka": "ka",
-      "kk": "kk",
-      "km": "km",
+      "ja": "jp",
       "kn": "kn",
       "ko": "ko",
-      "kok": "kok",
-      "ku": "ku",
-      "ky": "ky",
-      "lb": "lb",
-      "lo": "lo",
       "lt": "lt",
       "lv": "lv",
-      "mi": "mi",
-      "mk": "mk",
       "ml": "ml",
-      "mn": "MN",
       "mr": "mr",
       "ms": "ms",
-      "mt": "mt",
       "nb": "nb",
-      "ne": "ne",
       "nl": "nl",
-      "nn": "nn",
-      "or": "or",
       "pa": "pa",
-      "pa_Arab": "pa-Arab",
       "pl": "pl",
-      "pt": "pt",
-      "qu": "quz",
+      "pt": "pt-pt",
       "ro": "ro",
       "ru": "ru",
-      "rw": "rw",
-      "sd": "sd",
-      "si": "si",
       "sk": "sk",
       "sl": "sl",
-      "sq": "sq",
       "sr": "sr",
-      "sr_Latn": "sr_latn",
       "sv": "sv",
-      "sw": "sw",
       "ta": "ta",
       "te": "te",
-      "tg": "tg",
       "th": "th",
-      "ti": "ti",
-      "tk": "tk",
       "tr": "tr",
-      "tt": "tt",
-      "ug": "ug",
       "uk": "uk",
-      "ur": "ur",
-      "uz": "uz",
       "vi": "vi",
-      "wo": "wo",
-      "xh": "xh",
-      "yo": "yo",
-      "zh": "zh_Hans",
-      "zh_Hans": "zh_Hans",
-      "zh_Hant": "zh_Hant",
-      "zu": "zu"
+      "zh": "zh-hans",
+      "zh_Hans": "zh-hans",
+      "zh_Hant": "zh-hant"
     },
     "regions": {
-      "am-ET": "et",
-      "ar-AE": "ae",
-      "ar-BH": "bh",
-      "ar-DJ": "dj",
-      "ar-DZ": "dz",
-      "ar-EG": "eg",
-      "ar-ER": "er",
-      "ar-IL": "il",
-      "ar-IQ": "iq",
-      "ar-JO": "jo",
-      "ar-KM": "km",
-      "ar-KW": "kw",
-      "ar-LB": "lb",
-      "ar-LY": "ly",
-      "ar-MA": "ma",
-      "ar-MR": "mr",
-      "ar-OM": "om",
-      "ar-PS": "ps",
-      "ar-QA": "qa",
-      "ar-SA": "sa",
-      "ar-SD": "sd",
-      "ar-SO": "so",
-      "ar-SY": "sy",
-      "ar-TD": "td",
-      "ar-TN": "tn",
-      "ar-YE": "ye",
-      "az-AZ": "az",
-      "be-BY": "by",
-      "bg-BG": "bg",
-      "bn-BD": "bd",
-      "bs-BA": "ba",
-      "ca-AD": "ad",
-      "cs-CZ": "cz",
-      "da-DK": "dk",
-      "de-AT": "at",
-      "de-BE": "be",
-      "de-CH": "ch",
-      "de-DE": "de",
-      "de-LI": "li",
-      "de-LU": "lu",
-      "el-CY": "cy",
-      "el-GR": "gr",
-      "en-AG": "ag",
-      "en-AI": "ai",
-      "en-AS": "as",
-      "en-AU": "au",
-      "en-BB": "bb",
-      "en-BI": "bi",
-      "en-BM": "bm",
-      "en-BS": "bs",
-      "en-BW": "bw",
-      "en-BZ": "bz",
-      "en-CA": "ca",
-      "en-CC": "cc",
-      "en-CK": "ck",
-      "en-CM": "cm",
-      "en-CX": "cx",
-      "en-DM": "dm",
-      "en-ER": "er",
-      "en-FJ": "fj",
-      "en-FK": "fk",
-      "en-FM": "fm",
-      "en-GB": "gb",
-      "en-GD": "gd",
-      "en-GG": "gg",
-      "en-GH": "gh",
-      "en-GI": "gi",
-      "en-GM": "gm",
-      "en-GU": "gu",
-      "en-GY": "gy",
-      "en-HK": "hk",
-      "en-IE": "ie",
-      "en-IN": "in",
-      "en-JE": "je",
-      "en-JM": "jm",
-      "en-KE": "ke",
-      "en-KI": "ki",
-      "en-KN": "kn",
-      "en-KY": "ky",
-      "en-LC": "lc",
-      "en-LR": "lr",
-      "en-LS": "ls",
-      "en-MG": "mg",
-      "en-MH": "mh",
-      "en-MP": "mp",
-      "en-MS": "ms",
-      "en-MT": "mt",
-      "en-MU": "mu",
-      "en-MW": "mw",
-      "en-NA": "na",
-      "en-NF": "nf",
-      "en-NG": "ng",
-      "en-NR": "nr",
-      "en-NU": "nu",
-      "en-NZ": "nz",
-      "en-PG": "pg",
-      "en-PH": "ph",
-      "en-PK": "pk",
-      "en-PN": "pn",
-      "en-PR": "pr",
-      "en-PW": "pw",
-      "en-RW": "rw",
-      "en-SB": "sb",
-      "en-SC": "sc",
-      "en-SD": "sd",
-      "en-SG": "sg",
-      "en-SH": "sh",
-      "en-SL": "sl",
-      "en-SS": "ss",
-      "en-SX": "sx",
-      "en-SZ": "sz",
-      "en-TC": "tc",
-      "en-TK": "tk",
-      "en-TO": "to",
-      "en-TT": "tt",
-      "en-TV": "tv",
-      "en-TZ": "tz",
-      "en-UG": "ug",
-      "en-US": "us",
-      "en-VC": "vc",
-      "en-VG": "vg",
-      "en-VI": "vi",
-      "en-VU": "vu",
-      "en-WS": "ws",
-      "en-ZA": "za",
-      "en-ZM": "zm",
-      "en-ZW": "zw",
-      "es-AR": "ar",
-      "es-BO": "bo",
-      "es-CL": "cl",
-      "es-CO": "co",
-      "es-CR": "cr",
-      "es-DO": "do",
-      "es-EC": "ec",
-      "es-ES": "es",
-      "es-GQ": "gq",
-      "es-GT": "gt",
-      "es-HN": "hn",
-      "es-MX": "mx",
-      "es-NI": "ni",
-      "es-PA": "pa",
-      "es-PE": "pe",
-      "es-PR": "pr",
-      "es-PY": "py",
-      "es-SV": "sv",
-      "es-UY": "uy",
-      "es-VE": "ve",
-      "et-EE": "ee",
-      "fa-AF": "af",
-      "fa-IR": "ir",
-      "fi-FI": "fi",
-      "fil-PH": "ph",
-      "fr-BE": "be",
-      "fr-BF": "bf",
-      "fr-BI": "bi",
-      "fr-BJ": "bj",
-      "fr-BL": "bl",
-      "fr-CA": "ca",
-      "fr-CD": "cd",
-      "fr-CF": "cf",
-      "fr-CG": "cg",
-      "fr-CH": "ch",
-      "fr-CI": "ci",
-      "fr-CM": "cm",
-      "fr-DJ": "dj",
-      "fr-DZ": "dz",
-      "fr-FR": "fr",
-      "fr-GA": "ga",
-      "fr-GF": "gf",
-      "fr-GN": "gn",
-      "fr-GP": "gp",
-      "fr-GQ": "gq",
-      "fr-HT": "ht",
-      "fr-KM": "km",
-      "fr-LU": "lu",
-      "fr-MA": "ma",
-      "fr-MC": "mc",
-      "fr-MF": "mf",
-      "fr-MG": "mg",
-      "fr-ML": "ml",
-      "fr-MQ": "mq",
-      "fr-MU": "mu",
-      "fr-NC": "nc",
-      "fr-NE": "ne",
-      "fr-PF": "pf",
-      "fr-PM": "pm",
-      "fr-RE": "re",
-      "fr-RW": "rw",
-      "fr-SC": "sc",
-      "fr-SN": "sn",
-      "fr-SY": "sy",
-      "fr-TD": "td",
-      "fr-TG": "tg",
-      "fr-TN": "tn",
-      "fr-VU": "vu",
-      "fr-WF": "wf",
-      "fr-YT": "yt",
-      "ga-IE": "ie",
-      "he-IL": "il",
-      "hi-IN": "in",
-      "hr-BA": "ba",
-      "hr-HR": "hr",
-      "hu-HU": "hu",
-      "hy-AM": "am",
-      "id-ID": "id",
-      "is-IS": "is",
-      "it-CH": "ch",
-      "it-IT": "it",
-      "it-SM": "sm",
-      "it-VA": "va",
-      "ja-JP": "jp",
-      "ka-GE": "ge",
-      "kk-KZ": "kz",
-      "km-KH": "kh",
-      "ko-KR": "kr",
-      "ky-KG": "kg",
-      "lb-LU": "lu",
-      "lo-LA": "la",
-      "lt-LT": "lt",
-      "lv-LV": "lv",
-      "mi-NZ": "nz",
-      "mk-MK": "mk",
-      "mn-MN": "mn",
-      "ms-BN": "bn",
-      "ms-MY": "my",
-      "ms-SG": "sg",
-      "mt-MT": "mt",
-      "nb-NO": "no",
-      "ne-NP": "np",
-      "nl-AW": "aw",
-      "nl-BE": "be",
-      "nl-CW": "cw",
-      "nl-NL": "nl",
-      "nl-SR": "sr",
-      "nl-SX": "sx",
-      "nn-NO": "no",
-      "pl-PL": "pl",
-      "pt-AO": "ao",
-      "pt-BR": "br",
-      "pt-CV": "cv",
-      "pt-GQ": "gq",
-      "pt-GW": "gw",
-      "pt-MO": "mo",
-      "pt-MZ": "mz",
-      "pt-PT": "pt",
-      "pt-ST": "st",
-      "qu-BO": "bo",
-      "qu-EC": "ec",
-      "qu-PE": "pe",
-      "ro-MD": "md",
-      "ro-RO": "ro",
-      "ru-BY": "by",
-      "ru-KG": "kg",
-      "ru-KZ": "kz",
-      "ru-RU": "ru",
-      "ru-UA": "ua",
-      "rw-RW": "rw",
-      "si-LK": "lk",
-      "sk-SK": "sk",
-      "sl-SI": "si",
-      "sq-AL": "al",
-      "sr-BA": "ba",
-      "sr-ME": "me",
-      "sr-RS": "rs",
-      "sv-FI": "fi",
-      "sv-SE": "se",
-      "sw-KE": "ke",
-      "sw-TZ": "tz",
-      "sw-UG": "ug",
-      "ta-LK": "lk",
-      "ta-SG": "sg",
-      "tg-TJ": "tj",
-      "th-TH": "th",
-      "ti-ER": "er",
-      "tk-TM": "tm",
-      "tr-CY": "cy",
-      "tr-TR": "tr",
-      "uk-UA": "ua",
-      "ur-PK": "pk",
-      "uz-UZ": "uz",
-      "vi-VN": "vn",
-      "wo-SN": "sn",
-      "yo-NG": "ng",
-      "zh-CN": "cn",
-      "zh-HK": "hk",
-      "zh-MO": "mo",
-      "zh-SG": "sg",
-      "zh-TW": "tw"
+      "da-DK": "da-DK",
+      "de-AT": "de-AT",
+      "de-CH": "de-CH",
+      "de-DE": "de-DE",
+      "en-AU": "en-AU",
+      "en-CA": "en-CA",
+      "en-GB": "en-GB",
+      "en-IN": "en-IN",
+      "en-MY": "en-MY",
+      "en-NZ": "en-NZ",
+      "en-PH": "en-PH",
+      "en-US": "en-US",
+      "en-ZA": "en-ZA",
+      "es-AR": "es-AR",
+      "es-CL": "es-CL",
+      "es-ES": "es-ES",
+      "es-MX": "es-MX",
+      "es-US": "es-US",
+      "fi-FI": "fi-FI",
+      "fr-BE": "fr-BE",
+      "fr-CA": "fr-CA",
+      "fr-CH": "fr-CH",
+      "fr-FR": "fr-FR",
+      "id-ID": "en-ID",
+      "it-IT": "it-IT",
+      "ja-JP": "ja-JP",
+      "ko-KR": "ko-KR",
+      "nb-NO": "no-NO",
+      "nl-BE": "nl-BE",
+      "nl-NL": "nl-NL",
+      "pl-PL": "pl-PL",
+      "pt-BR": "pt-BR",
+      "ru-RU": "ru-RU",
+      "sv-SE": "sv-SE",
+      "tr-TR": "tr-TR",
+      "zh-CN": "zh-CN",
+      "zh-HK": "zh-HK",
+      "zh-TW": "zh-TW"
     },
-    "supported_languages": [
-      "af",
-      "am",
-      "ar",
-      "as",
-      "az-latn",
-      "be",
-      "bg",
-      "bn",
-      "bs-latn",
-      "ca",
-      "ca-es-valencia",
-      "chr-cher",
-      "cs",
-      "cy",
-      "da",
-      "de",
-      "el",
-      "en",
-      "es",
-      "et",
-      "eu",
-      "fa",
-      "fi",
-      "fil",
-      "fr",
-      "ga",
-      "gd",
-      "gl",
-      "gu",
-      "ha-latn",
-      "he",
-      "hi",
-      "hr",
-      "hu",
-      "hy",
-      "id",
-      "ig",
-      "is",
-      "it",
-      "ja",
-      "ka",
-      "kk",
-      "km",
-      "kn",
-      "ko",
-      "kok",
-      "ku-arab",
-      "ky",
-      "lb",
-      "lo",
-      "lt",
-      "lv",
-      "mi",
-      "mk",
-      "ml",
-      "mn-Cyrl-MN",
-      "mr",
-      "ms",
-      "mt",
-      "nb",
-      "ne",
-      "nl",
-      "nn",
-      "nso",
-      "or",
-      "pa-arab",
-      "pa-guru",
-      "pl",
-      "prs",
-      "pt-BR",
-      "pt-PT",
-      "quc",
-      "quz",
-      "ro",
-      "ru",
-      "rw",
-      "sd-arab",
-      "si",
-      "sk",
-      "sl",
-      "sq",
-      "sr-cyrl",
-      "sr-latn",
-      "sv",
-      "sw",
-      "ta",
-      "te",
-      "tg-cyrl",
-      "th",
-      "ti",
-      "tk",
-      "tn",
-      "tr",
-      "tt",
-      "ug",
-      "uk",
-      "ur",
-      "uz-latn",
-      "vi",
-      "wo",
-      "xh",
-      "yo",
-      "zh-Hans",
-      "zh-Hant",
-      "zu"
-    ]
+    "supported_languages": {}
   },
   "dailymotion": {
     "all_locale": null,

searx/engines/bing.py

@@ -4,16 +4,28 @@
 
 - https://github.com/searx/searx/issues/2019#issuecomment-648227442
 """
-# pylint: disable=too-many-branches
 
+# pylint: disable=too-many-branches, invalid-name
+
+from typing import TYPE_CHECKING
+import datetime
 import re
-from urllib.parse import urlencode, urlparse, parse_qs
+import uuid
+from urllib.parse import urlencode
 from lxml import html
-from searx.utils import eval_xpath, extract_text, eval_xpath_list, match_language, eval_xpath_getindex
-from searx.network import multi_requests, Request
+import babel
+import babel.languages
 
+from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_getindex
+from searx import network
+from searx.locales import language_tag, region_tag
 from searx.enginelib.traits import EngineTraits
 
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
+
 traits: EngineTraits
 
 about = {
@@ -25,56 +37,124 @@ about = {
     "results": 'HTML',
 }
 
+send_accept_language_header = True
+"""Bing tries to guess user's language and territory from the HTTP
+Accept-Language.  Optional the user can select a search-language (can be
+different to the UI language) and a region (market code)."""
+
 # engine dependent config
 categories = ['general', 'web']
 paging = True
-time_range_support = False
-safesearch = False
-send_accept_language_header = True
-supported_languages_url = 'https://www.bing.com/account/general'
-language_aliases = {}
-
-# search-url
-base_url = 'https://www.bing.com/'
+time_range_support = True
+safesearch = True
+safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'}  # cookie: ADLT=STRICT
 
-# initial query:     https://www.bing.com/search?q=foo&search=&form=QBLH
-inital_query = 'search?{query}&search=&form=QBLH'
+base_url = 'https://www.bing.com/search'
+"""Bing (Web) search URL"""
 
-# following queries: https://www.bing.com/search?q=foo&search=&first=11&FORM=PERE
-page_query = 'search?{query}&search=&first={offset}&FORM=PERE'
+bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes'
+"""Bing (Web) search API description"""
 
 
 def _get_offset_from_pageno(pageno):
     return (pageno - 1) * 10 + 1
 
 
-def request(query, params):
+def set_bing_cookies(params, engine_language, engine_region, SID):
+
+    # set cookies
+    # -----------
+
+    params['cookies']['_EDGE_V'] = '1'
+
+    # _EDGE_S: F=1&SID=3A5253BD6BCA609509B741876AF961CA&mkt=zh-tw
+    _EDGE_S = [
+        'F=1',
+        'SID=%s' % SID,
+        'mkt=%s' % engine_region.lower(),
+        'ui=%s' % engine_language.lower(),
+    ]
+    params['cookies']['_EDGE_S'] = '&'.join(_EDGE_S)
+    logger.debug("cookie _EDGE_S=%s", params['cookies']['_EDGE_S'])
+
+    # "_EDGE_CD": "m=zh-tw",
 
-    offset = _get_offset_from_pageno(params.get('pageno', 1))
+    _EDGE_CD = [  # pylint: disable=invalid-name
+        'm=%s' % engine_region.lower(),  # search region: zh-cn
+        'u=%s' % engine_language.lower(),  # UI: en-us
+    ]
+
+    params['cookies']['_EDGE_CD'] = '&'.join(_EDGE_CD) + ';'
+    logger.debug("cookie _EDGE_CD=%s", params['cookies']['_EDGE_CD'])
 
-    # logger.debug("params['pageno'] --> %s", params.get('pageno'))
-    # logger.debug("          offset --> %s", offset)
+    SRCHHPGUSR = [  # pylint: disable=invalid-name
+        'SRCHLANG=%s' % engine_language,
+        # Trying to set ADLT cookie here seems not to have any effect, I assume
+        # there is some age verification by a cookie (and/or session ID) needed,
+        # to disable the SafeSearch.
+        'ADLT=%s' % safesearch_types.get(params['safesearch'], 'DEMOTE'),
+    ]
+    params['cookies']['SRCHHPGUSR'] = '&'.join(SRCHHPGUSR)
+    logger.debug("cookie SRCHHPGUSR=%s", params['cookies']['SRCHHPGUSR'])
 
-    search_string = page_query
-    if offset == 1:
-        search_string = inital_query
 
-    if params['language'] == 'all':
-        lang = 'EN'
-    else:
-        lang = match_language(params['language'], supported_languages, language_aliases)
+def request(query, params):
+    """Assemble a Bing-Web request."""
 
-    query = 'language:{} {}'.format(lang.split('-')[0].upper(), query)
+    engine_region = traits.get_region(params['searxng_locale'], 'en-US')
+    engine_language = traits.get_language(params['searxng_locale'], 'en')
 
-    search_path = search_string.format(query=urlencode({'q': query}), offset=offset)
+    SID = uuid.uuid1().hex.upper()
+    CVID = uuid.uuid1().hex.upper()
 
-    if offset > 1:
-        referer = base_url + inital_query.format(query=urlencode({'q': query}))
+    set_bing_cookies(params, engine_language, engine_region, SID)
+
+    # build URL query
+    # ---------------
+
+    # query term
+    page = int(params.get('pageno', 1))
+    query_params = {
+        # fmt: off
+        'q': query,
+        'pq': query,
+        'cvid': CVID,
+        'qs': 'n',
+        'sp': '-1'
+        # fmt: on
+    }
+
+    # page
+    if page > 1:
+        referer = base_url + '?' + urlencode(query_params)
         params['headers']['Referer'] = referer
         logger.debug("headers.Referer --> %s", referer)
 
-    params['url'] = base_url + search_path
-    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+    query_params['first'] = _get_offset_from_pageno(page)
+
+    if page == 2:
+        query_params['FORM'] = 'PERE'
+    elif page > 2:
+        query_params['FORM'] = 'PERE%s' % (page - 2)
+
+    filters = ''
+    if params['time_range']:
+        query_params['filt'] = 'custom'
+
+        if params['time_range'] == 'day':
+            filters = 'ex1:"ez1"'
+        elif params['time_range'] == 'week':
+            filters = 'ex1:"ez2"'
+        elif params['time_range'] == 'month':
+            filters = 'ex1:"ez3"'
+        elif params['time_range'] == 'year':
+            epoch_1970 = datetime.date(1970, 1, 1)
+            today_no = (datetime.date.today() - epoch_1970).days
+            filters = 'ex1:"ez5_%s_%s"' % (today_no - 365, today_no)
+
+    params['url'] = base_url + '?' + urlencode(query_params)
+    if filters:
+        params['url'] = params['url'] + '&filters=' + filters
     return params
 
 
@@ -111,7 +191,8 @@ def response(resp):
             url_cite = extract_text(eval_xpath(result, './/div[@class="b_attribution"]/cite'))
             # Bing can shorten the URL either at the end or in the middle of the string
             if (
-                url_cite.startswith('https://')
+                url_cite
+                and url_cite.startswith('https://')
                 and '…' not in url_cite
                 and '...' not in url_cite
                 and '›' not in url_cite
@@ -131,9 +212,9 @@ def response(resp):
 
     # resolve all Bing redirections in parallel
     request_list = [
-        Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve
+        network.Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve
     ]
-    response_list = multi_requests(request_list)
+    response_list = network.multi_requests(request_list)
     for i, redirect_response in enumerate(response_list):
         if not isinstance(redirect_response, Exception):
             results[url_to_resolve_index[i]]['url'] = redirect_response.headers['location']
@@ -161,94 +242,43 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
-
-    lang_tags = set()
-
-    dom = html.fromstring(resp.text)
-    lang_links = eval_xpath(dom, '//div[@id="language-section"]//li')
-
-    for _li in lang_links:
-
-        href = eval_xpath(_li, './/@href')[0]
-        (_scheme, _netloc, _path, _params, query, _fragment) = urlparse(href)
-        query = parse_qs(query, keep_blank_values=True)
-
-        # fmt: off
-        setlang = query.get('setlang', [None, ])[0]
-        # example: 'mn-Cyrl-MN' --> '['mn', 'Cyrl-MN']
-        lang, nation = (setlang.split('-', maxsplit=1) + [None,])[:2]  # fmt: skip
-        # fmt: on
-
-        tag = lang + '-' + nation if nation else lang
-        lang_tags.add(tag)
-
-    return list(lang_tags)
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-Web."""
 
+    xpath_market_codes = '//table[1]/tbody/tr/td[3]'
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
 
-def fetch_traits(engine_traits: EngineTraits):
-    """Fetch languages and regions from bing."""
+    _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes)
 
-    # pylint: disable=import-outside-toplevel, disable=too-many-branches,
-    # pylint: disable=too-many-locals, too-many-statements
 
-    engine_traits.data_type = 'supported_languages'  # deprecated
+def _fetch_traits(engine_traits: EngineTraits, url: str, xpath_language_codes: str, xpath_market_codes: str):
 
-    import babel
-    import babel.languages
-    from searx import network
-    from searx.locales import get_offical_locales, language_tag, region_tag
-    from searx.utils import gen_useragent
+    # insert alias to map from a language (zh) to a language + script (zh_Hans)
+    engine_traits.languages['zh'] = 'zh-hans'
 
-    headers = {
-        'User-Agent': gen_useragent(),
-        'Accept-Language': "en-US,en;q=0.5",  # bing needs to set the English language
-    }
-    resp = network.get('https://www.bing.com/account/general', headers=headers)
+    resp = network.get(url)
 
     if not resp.ok:
         print("ERROR: response from peertube is not OK.")
 
     dom = html.fromstring(resp.text)
 
-    # Selector to get items from "Display language"
+    map_lang = {'jp': 'ja'}
+    for td in eval_xpath(dom, xpath_language_codes):
+        eng_lang = td.text
 
-    lang_map = {
-        'prs': 'fa',  # Persian
-        'pt_BR': 'pt',  # Portuguese (Brasil)
-        'pt_PT': 'pt',  # Portuguese (Portugal)
-        'ca-ES-VALENCIA': 'ca',  # Catalan (Spain, Valencian)
-    }
-
-    unknow_langs = [
-        'quc',  # K'iche'
-        'nso',  # Sesotho sa Leboa
-        'tn',  # Setswana
-    ]
-
-    for div in eval_xpath(dom, '//div[@id="limit-languages"]//input/..'):
-
-        eng_lang = eval_xpath(div, './/input/@value')[0]
-        if eng_lang in unknow_langs:
+        if eng_lang in ('en-gb', 'pt-br'):
+            # language 'en' is already in the list and a language 'en-gb' can't
+            # be handled in SearXNG, same with pt-br which is covered by pt-pt.
             continue
 
-        eng_lang = lang_map.get(eng_lang, eng_lang)
-        label = extract_text(eval_xpath(div, './/label'))
-
-        # The 'language:xx' query string in the request function (above) does
-        # only support the language codes from the "Display languages" list.
-        # Examples of items from the "Display languages" not sopported in the
-        # query string: zh_Hans --> zh / sr_latn --> sr
-        #
-        # eng_lang = eng_lang.split('_')[0]
-
+        babel_lang = map_lang.get(eng_lang, eng_lang).replace('-', '_')
         try:
-            sxng_tag = language_tag(babel.Locale.parse(eng_lang.replace('-', '_'), sep='_'))
+            sxng_tag = language_tag(babel.Locale.parse(babel_lang))
         except babel.UnknownLocaleError:
-            print("ERROR: %s (%s) is unknown by babel" % (label, eng_lang))
+            print("ERROR: language (%s) is unknown by babel" % (eng_lang))
             continue
-
         conflict = engine_traits.languages.get(sxng_tag)
         if conflict:
             if conflict != eng_lang:
@@ -256,25 +286,27 @@ def fetch_traits(engine_traits: EngineTraits):
             continue
         engine_traits.languages[sxng_tag] = eng_lang
 
-    engine_traits.languages['zh'] = 'zh_Hans'
+    map_region = {
+        'en-ID': 'id_ID',
+        'no-NO': 'nb_NO',
+    }
 
-    # regiones
+    for td in eval_xpath(dom, xpath_market_codes):
+        eng_region = td.text
+        babel_region = map_region.get(eng_region, eng_region).replace('-', '_')
 
-    for a in eval_xpath(dom, '//div[@id="region-section-content"]//li/a'):
-        href = eval_xpath(a, './/@href')[0]
-        # lang_name = extract_text(a)
-        query = urlparse(href)[4]
-        query = parse_qs(query, keep_blank_values=True)
-        cc = query.get('cc')[0]  # pylint:disable=invalid-name
-        if cc == 'clear':
+        if eng_region == 'en-WW':
+            engine_traits.all_locale = eng_region
             continue
 
-        # Assert babel supports this locales
-        sxng_locales = get_offical_locales(cc.upper(), engine_traits.languages.keys())
-
-        if not sxng_locales:
-            # print("ERROR: can't map from bing country %s (%s) to a babel region." % (a.text_content().strip(), cc))
+        try:
+            sxng_tag = region_tag(babel.Locale.parse(babel_region))
+        except babel.UnknownLocaleError:
+            print("ERROR: region (%s) is unknown by babel" % (eng_region))
             continue
-
-        for sxng_locale in sxng_locales:
-            engine_traits.regions[region_tag(sxng_locale)] = cc
+        conflict = engine_traits.regions.get(sxng_tag)
+        if conflict:
+            if conflict != eng_region:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_region))
+            continue
+        engine_traits.regions[sxng_tag] = eng_region

searx/engines/bing_images.py

@@ -10,12 +10,7 @@ from urllib.parse import urlencode
 from lxml import html
 
 from searx.utils import match_language
-from searx.engines.bing import language_aliases
-from searx.engines.bing import (  # pylint: disable=unused-import
-    _fetch_supported_languages,
-    fetch_traits,
-    supported_languages_url,
-)
+from searx.enginelib.traits import EngineTraits
 
 # about
 about = {
@@ -106,3 +101,19 @@ def response(resp):
         )
 
     return results
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-News."""
+    # pylint: disable=import-outside-toplevel
+
+    from searx.engines.bing import _fetch_traits
+
+    url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-news-search/reference/market-codes'
+
+    xpath_market_codes = '//table[1]/tbody/tr/td[3]'
+
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
+
+    _fetch_traits(engine_traits, url, xpath_language_codes, xpath_market_codes)

searx/engines/bing_news.py

@@ -14,12 +14,7 @@ from dateutil import parser
 from lxml import etree
 from lxml.etree import XPath
 from searx.utils import match_language, eval_xpath_getindex
-from searx.engines.bing import (  # pylint: disable=unused-import
-    language_aliases,
-    _fetch_supported_languages,
-    fetch_traits,
-    supported_languages_url,
-)
+from searx.enginelib.traits import EngineTraits
 
 # about
 about = {
@@ -140,3 +135,21 @@ def response(resp):
             results.append({'url': url, 'title': title, 'publishedDate': publishedDate, 'content': content})
 
     return results
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-News."""
+    # pylint: disable=import-outside-toplevel
+
+    from searx.engines.bing import _fetch_traits
+
+    url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-news-search/reference/market-codes'
+
+    # The description of the first table says "query parameter when calling the
+    # Video Search API." .. thats why I use the 4. table "News Category API markets"
+    xpath_market_codes = '//table[4]/tbody/tr/td[3]'
+
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
+
+    _fetch_traits(engine_traits, url, xpath_language_codes, xpath_market_codes)

searx/engines/bing_videos.py

@@ -10,13 +10,7 @@ from urllib.parse import urlencode
 from lxml import html
 
 from searx.utils import match_language
-from searx.engines.bing import language_aliases
-
-from searx.engines.bing import (  # pylint: disable=unused-import
-    _fetch_supported_languages,
-    fetch_traits,
-    supported_languages_url,
-)
+from searx.enginelib.traits import EngineTraits
 
 about = {
     "website": 'https://www.bing.com/videos',
@@ -97,3 +91,17 @@ def response(resp):
         )
 
     return results
+
+
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages and regions from Bing-Videos."""
+    # pylint: disable=import-outside-toplevel
+
+    from searx.engines.bing import _fetch_traits
+
+    url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-video-search/reference/market-codes'
+    xpath_market_codes = '//table[1]/tbody/tr/td[3]'
+    # xpath_country_codes = '//table[2]/tbody/tr/td[2]'
+    xpath_language_codes = '//table[3]/tbody/tr/td[2]'
+
+    _fetch_traits(engine_traits, url, xpath_language_codes, xpath_market_codes)

searx/sxng_locales.py

@@ -8,8 +8,6 @@ This file is generated automatically by::
 
 sxng_locales = (
     ('ar', 'العربية', '', 'Arabic', '\U0001f310'),
-    ('ar-EG', 'العربية', 'مصر', 'Arabic', '\U0001f1ea\U0001f1ec'),
-    ('ar-SA', 'العربية', 'المملكة العربية السعودية', 'Arabic', '\U0001f1f8\U0001f1e6'),
     ('bg', 'Български', '', 'Bulgarian', '\U0001f310'),
     ('bg-BG', 'Български', 'България', 'Bulgarian', '\U0001f1e7\U0001f1ec'),
     ('ca', 'Català', '', 'Catalan', '\U0001f310'),
@@ -30,19 +28,17 @@ sxng_locales = (
     ('en-GB', 'English', 'United Kingdom', 'English', '\U0001f1ec\U0001f1e7'),
     ('en-IE', 'English', 'Ireland', 'English', '\U0001f1ee\U0001f1ea'),
     ('en-IN', 'English', 'India', 'English', '\U0001f1ee\U0001f1f3'),
+    ('en-MY', 'English', 'Malaysia', 'English', '\U0001f1f2\U0001f1fe'),
     ('en-NZ', 'English', 'New Zealand', 'English', '\U0001f1f3\U0001f1ff'),
     ('en-PH', 'English', 'Philippines', 'English', '\U0001f1f5\U0001f1ed'),
-    ('en-PK', 'English', 'Pakistan', 'English', '\U0001f1f5\U0001f1f0'),
-    ('en-SG', 'English', 'Singapore', 'English', '\U0001f1f8\U0001f1ec'),
     ('en-US', 'English', 'United States', 'English', '\U0001f1fa\U0001f1f8'),
     ('en-ZA', 'English', 'South Africa', 'English', '\U0001f1ff\U0001f1e6'),
     ('es', 'Español', '', 'Spanish', '\U0001f310'),
     ('es-AR', 'Español', 'Argentina', 'Spanish', '\U0001f1e6\U0001f1f7'),
     ('es-CL', 'Español', 'Chile', 'Spanish', '\U0001f1e8\U0001f1f1'),
-    ('es-CO', 'Español', 'Colombia', 'Spanish', '\U0001f1e8\U0001f1f4'),
     ('es-ES', 'Español', 'España', 'Spanish', '\U0001f1ea\U0001f1f8'),
     ('es-MX', 'Español', 'México', 'Spanish', '\U0001f1f2\U0001f1fd'),
-    ('es-PE', 'Español', 'Perú', 'Spanish', '\U0001f1f5\U0001f1ea'),
+    ('es-US', 'Español', 'Estados Unidos', 'Spanish', '\U0001f1fa\U0001f1f8'),
     ('et', 'Eesti', '', 'Estonian', '\U0001f310'),
     ('et-EE', 'Eesti', 'Eesti', 'Estonian', '\U0001f1ea\U0001f1ea'),
     ('fi', 'Suomi', '', 'Finnish', '\U0001f310'),
@@ -53,8 +49,6 @@ sxng_locales = (
     ('fr-CH', 'Français', 'Suisse', 'French', '\U0001f1e8\U0001f1ed'),
     ('fr-FR', 'Français', 'France', 'French', '\U0001f1eb\U0001f1f7'),
     ('he', 'עברית', '', 'Hebrew', '\U0001f1ee\U0001f1f7'),
-    ('hr', 'Hrvatski', '', 'Croatian', '\U0001f310'),
-    ('hr-HR', 'Hrvatski', 'Hrvatska', 'Croatian', '\U0001f1ed\U0001f1f7'),
     ('hu', 'Magyar', '', 'Hungarian', '\U0001f310'),
     ('hu-HU', 'Magyar', 'Magyarország', 'Hungarian', '\U0001f1ed\U0001f1fa'),
     ('id', 'Indonesia', '', 'Indonesian', '\U0001f310'),
@@ -67,9 +61,7 @@ sxng_locales = (
     ('ko', '한국어', '', 'Korean', '\U0001f310'),
     ('ko-KR', '한국어', '대한민국', 'Korean', '\U0001f1f0\U0001f1f7'),
     ('lt', 'Lietuvių', '', 'Lithuanian', '\U0001f310'),
-    ('lt-LT', 'Lietuvių', 'Lietuva', 'Lithuanian', '\U0001f1f1\U0001f1f9'),
     ('lv', 'Latviešu', '', 'Latvian', '\U0001f310'),
-    ('lv-LV', 'Latviešu', 'Latvija', 'Latvian', '\U0001f1f1\U0001f1fb'),
     ('nb', 'Norsk Bokmål', '', 'Norwegian Bokmål', '\U0001f310'),
     ('nb-NO', 'Norsk Bokmål', 'Norge', 'Norwegian Bokmål', '\U0001f1f3\U0001f1f4'),
     ('nl', 'Nederlands', '', 'Dutch', '\U0001f310'),
@@ -85,19 +77,13 @@ sxng_locales = (
     ('ru', 'Русский', '', 'Russian', '\U0001f310'),
     ('ru-RU', 'Русский', 'Россия', 'Russian', '\U0001f1f7\U0001f1fa'),
     ('sk', 'Slovenčina', '', 'Slovak', '\U0001f310'),
-    ('sk-SK', 'Slovenčina', 'Slovensko', 'Slovak', '\U0001f1f8\U0001f1f0'),
     ('sl', 'Slovenščina', '', 'Slovenian', '\U0001f310'),
-    ('sl-SI', 'Slovenščina', 'Slovenija', 'Slovenian', '\U0001f1f8\U0001f1ee'),
     ('sv', 'Svenska', '', 'Swedish', '\U0001f310'),
     ('sv-SE', 'Svenska', 'Sverige', 'Swedish', '\U0001f1f8\U0001f1ea'),
     ('th', 'ไทย', '', 'Thai', '\U0001f310'),
     ('th-TH', 'ไทย', 'ไทย', 'Thai', '\U0001f1f9\U0001f1ed'),
     ('tr', 'Türkçe', '', 'Turkish', '\U0001f310'),
     ('tr-TR', 'Türkçe', 'Türkiye', 'Turkish', '\U0001f1f9\U0001f1f7'),
-    ('uk', 'Українська', '', 'Ukrainian', '\U0001f310'),
-    ('uk-UA', 'Українська', 'Україна', 'Ukrainian', '\U0001f1fa\U0001f1e6'),
-    ('vi', 'Tiếng Việt', '', 'Vietnamese', '\U0001f310'),
-    ('vi-VN', 'Tiếng Việt', 'Việt Nam', 'Vietnamese', '\U0001f1fb\U0001f1f3'),
     ('zh', '中文', '', 'Chinese', '\U0001f310'),
     ('zh-CN', '中文', '中国', 'Chinese', '\U0001f1e8\U0001f1f3'),
     ('zh-HK', '中文', '中國香港', 'Chinese', '\U0001f1ed\U0001f1f0'),