__init__.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. # SPDX-License-Identifier: AGPL-3.0-or-later
  2. # lint: pylint
  3. """This module implements the engine loader.
  4. Load and initialize the ``engines``, see :py:func:`load_engines` and register
  5. :py:obj:`engine_shortcuts`.
  6. usage::
  7. load_engines( settings['engines'] )
  8. """
  9. import sys
  10. import copy
  11. from os.path import realpath, dirname
  12. from babel.localedata import locale_identifiers
  13. from searx import logger, settings
  14. from searx.data import ENGINES_LANGUAGES
  15. from searx.network import get
  16. from searx.utils import load_module, match_language, gen_useragent
  17. logger = logger.getChild('engines')
  18. ENGINE_DIR = dirname(realpath(__file__))
  19. BABEL_LANGS = [
  20. lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]
  21. for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())
  22. ]
  23. ENGINE_DEFAULT_ARGS = {
  24. "engine_type": "online",
  25. "inactive": False,
  26. "disabled": False,
  27. "timeout": settings["outgoing"]["request_timeout"],
  28. "shortcut": "-",
  29. "categories": ["general"],
  30. "supported_languages": [],
  31. "language_aliases": {},
  32. "paging": False,
  33. "safesearch": False,
  34. "time_range_support": False,
  35. "enable_http": False,
  36. "display_error_messages": True,
  37. "tokens": [],
  38. }
  39. """Defaults for the namespace of an engine module, see :py:func:`load_engine`"""
  40. categories = {'general': []}
  41. engines = {}
  42. engine_shortcuts = {}
  43. """Simple map of registered *shortcuts* to name of the engine (or ``None``).
  44. ::
  45. engine_shortcuts[engine.shortcut] = engine.name
  46. """
  47. def load_engine(engine_data):
  48. """Load engine from ``engine_data``.
  49. :param dict engine_data: Attributes from YAML ``settings:engines/<engine>``
  50. :return: initialized namespace of the ``<engine>``.
  51. 1. create a namespace and load module of the ``<engine>``
  52. 2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS`
  53. 3. update namespace with values from ``engine_data``
  54. If engine *is active*, return namespace of the engine, otherwise return
  55. ``None``.
  56. This function also returns ``None`` if initialization of the namespace fails
  57. for one of the following reasons:
  58. - engine name contains underscore
  59. - engine name is not lowercase
  60. - required attribute is not set :py:func:`is_missing_required_attributes`
  61. """
  62. engine_name = engine_data['name']
  63. if '_' in engine_name:
  64. logger.error('Engine name contains underscore: "{}"'.format(engine_name))
  65. return None
  66. if engine_name.lower() != engine_name:
  67. logger.warn('Engine name is not lowercase: "{}", converting to lowercase'.format(engine_name))
  68. engine_name = engine_name.lower()
  69. engine_data['name'] = engine_name
  70. # load_module
  71. engine_module = engine_data['engine']
  72. try:
  73. engine = load_module(engine_module + '.py', ENGINE_DIR)
  74. except (SyntaxError, KeyboardInterrupt, SystemExit, SystemError, ImportError, RuntimeError):
  75. logger.exception('Fatal exception in engine "{}"'.format(engine_module))
  76. sys.exit(1)
  77. except BaseException:
  78. logger.exception('Cannot load engine "{}"'.format(engine_module))
  79. return None
  80. update_engine_attributes(engine, engine_data)
  81. set_language_attributes(engine)
  82. update_attributes_for_tor(engine)
  83. if not is_engine_active(engine):
  84. return None
  85. if is_missing_required_attributes(engine):
  86. return None
  87. set_loggers(engine, engine_name)
  88. return engine
  89. def set_loggers(engine, engine_name):
  90. # set the logger for engine
  91. engine.logger = logger.getChild(engine_name)
  92. # the engine may have load some other engines
  93. # may sure the logger is initialized
  94. for module_name, module in sys.modules.items():
  95. if (
  96. module_name.startswith("searx.engines")
  97. and module_name != "searx.engines.__init__"
  98. and not hasattr(module, "logger")
  99. ):
  100. module_engine_name = module_name.split(".")[-1]
  101. module.logger = logger.getChild(module_engine_name)
  102. def update_engine_attributes(engine, engine_data):
  103. # set engine attributes from engine_data
  104. for param_name, param_value in engine_data.items():
  105. if param_name == 'categories':
  106. if isinstance(param_value, str):
  107. param_value = list(map(str.strip, param_value.split(',')))
  108. engine.categories = param_value
  109. else:
  110. setattr(engine, param_name, param_value)
  111. # set default attributes
  112. for arg_name, arg_value in ENGINE_DEFAULT_ARGS.items():
  113. if not hasattr(engine, arg_name):
  114. setattr(engine, arg_name, copy.deepcopy(arg_value))
  115. def set_language_attributes(engine):
  116. # assign supported languages from json file
  117. if engine.name in ENGINES_LANGUAGES:
  118. engine.supported_languages = ENGINES_LANGUAGES[engine.name]
  119. elif engine.engine in ENGINES_LANGUAGES:
  120. # The key of the dictionary ENGINES_LANGUAGES is the *engine name*
  121. # configured in settings.xml. When multiple engines are configured in
  122. # settings.yml to use the same origin engine (python module) these
  123. # additional engines can use the languages from the origin engine.
  124. # For this use the configured ``engine: ...`` from settings.yml
  125. engine.supported_languages = ENGINES_LANGUAGES[engine.engine]
  126. if hasattr(engine, 'language'):
  127. # For an engine, when there is `language: ...` in the YAML settings, the
  128. # engine supports only one language, in this case
  129. # engine.supported_languages should contains this value defined in
  130. # settings.yml
  131. if engine.language not in engine.supported_languages:
  132. raise ValueError(
  133. "settings.yml - engine: '%s' / language: '%s' not supported" % (
  134. engine.name, engine.language ))
  135. if isinstance(engine.supported_languages, dict):
  136. engine.supported_languages = {
  137. engine.language : engine.supported_languages[engine.language]
  138. }
  139. else:
  140. engine.supported_languages = [engine.language]
  141. # find custom aliases for non standard language codes
  142. for engine_lang in engine.supported_languages:
  143. iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None)
  144. if (iso_lang
  145. and iso_lang != engine_lang
  146. and not engine_lang.startswith(iso_lang)
  147. and iso_lang not in engine.supported_languages
  148. ):
  149. engine.language_aliases[iso_lang] = engine_lang
  150. # language_support
  151. engine.language_support = len(engine.supported_languages) > 0
  152. # assign language fetching method if auxiliary method exists
  153. if hasattr(engine, '_fetch_supported_languages'):
  154. headers = {
  155. 'User-Agent': gen_useragent(),
  156. 'Accept-Language': 'ja-JP,ja;q=0.8,en-US;q=0.5,en;q=0.3', # bing needs a non-English language
  157. }
  158. engine.fetch_supported_languages = (
  159. # pylint: disable=protected-access
  160. lambda: engine._fetch_supported_languages(
  161. get(engine.supported_languages_url, headers=headers))
  162. )
  163. def update_attributes_for_tor(engine):
  164. if (settings['outgoing'].get('using_tor_proxy')
  165. and hasattr(engine, 'onion_url') ):
  166. engine.search_url = engine.onion_url + getattr(engine, 'search_path', '')
  167. engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0)
  168. def is_missing_required_attributes(engine):
  169. """An attribute is required when its name doesn't start with ``_`` (underline).
  170. Required attributes must not be ``None``.
  171. """
  172. missing = False
  173. for engine_attr in dir(engine):
  174. if not engine_attr.startswith('_') and getattr(engine, engine_attr) is None:
  175. logger.error(
  176. 'Missing engine config attribute: "{0}.{1}"'
  177. .format(engine.name, engine_attr))
  178. missing = True
  179. return missing
  180. def is_engine_active(engine):
  181. # check if engine is inactive
  182. if engine.inactive is True:
  183. return False
  184. # exclude onion engines if not using tor
  185. if ('onions' in engine.categories
  186. and not settings['outgoing'].get('using_tor_proxy') ):
  187. return False
  188. return True
  189. def register_engine(engine):
  190. if engine.name in engines:
  191. logger.error('Engine config error: ambigious name: {0}'.format(engine.name))
  192. sys.exit(1)
  193. engines[engine.name] = engine
  194. if engine.shortcut in engine_shortcuts:
  195. logger.error('Engine config error: ambigious shortcut: {0}'.format(engine.shortcut))
  196. sys.exit(1)
  197. engine_shortcuts[engine.shortcut] = engine.name
  198. for category_name in engine.categories:
  199. categories.setdefault(category_name, []).append(engine)
  200. def load_engines(engine_list):
  201. """usage: ``engine_list = settings['engines']``
  202. """
  203. engines.clear()
  204. engine_shortcuts.clear()
  205. categories.clear()
  206. categories['general'] = []
  207. for engine_data in engine_list:
  208. engine = load_engine(engine_data)
  209. if engine:
  210. register_engine(engine)
  211. return engines