elasticsearch.py 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177
  1. # SPDX-License-Identifier: AGPL-3.0-or-later
  2. """.. sidebar:: info
  3. - :origin:`elasticsearch.py <searx/engines/elasticsearch.py>`
  4. - `Elasticsearch <https://www.elastic.co/elasticsearch/>`_
  5. - `Elasticsearch Guide
  6. <https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html>`_
  7. - `Install Elasticsearch
  8. <https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html>`_
  9. Elasticsearch_ supports numerous ways to query the data it is storing. At the
  10. moment the engine supports the most popular search methods (``query_type``):
  11. - ``match``,
  12. - ``simple_query_string``,
  13. - ``term`` and
  14. - ``terms``.
  15. If none of the methods fit your use case, you can select ``custom`` query type
  16. and provide the JSON payload to submit to Elasticsearch in
  17. ``custom_query_json``.
  18. Example
  19. =======
  20. The following is an example configuration for an Elasticsearch_ instance with
  21. authentication configured to read from ``my-index`` index.
  22. .. code:: yaml
  23. - name: elasticsearch
  24. shortcut: es
  25. engine: elasticsearch
  26. base_url: http://localhost:9200
  27. username: elastic
  28. password: changeme
  29. index: my-index
  30. query_type: match
  31. # custom_query_json: '{ ... }'
  32. enable_http: true
  33. """
  34. from json import loads, dumps
  35. from searx.exceptions import SearxEngineAPIException
  36. from searx.result_types import EngineResults
  37. from searx.extended_types import SXNG_Response
  38. base_url = 'http://localhost:9200'
  39. username = ''
  40. password = ''
  41. index = ''
  42. search_url = '{base_url}/{index}/_search'
  43. query_type = 'match'
  44. custom_query_json = {}
  45. show_metadata = False
  46. categories = ['general']
  47. def init(engine_settings):
  48. if 'query_type' in engine_settings and engine_settings['query_type'] not in _available_query_types:
  49. raise ValueError('unsupported query type', engine_settings['query_type'])
  50. if index == '':
  51. raise ValueError('index cannot be empty')
  52. def request(query, params):
  53. if query_type not in _available_query_types:
  54. return params
  55. if username and password:
  56. params['auth'] = (username, password)
  57. params['url'] = search_url.format(base_url=base_url, index=index)
  58. params['method'] = 'GET'
  59. params['data'] = dumps(_available_query_types[query_type](query))
  60. params['headers']['Content-Type'] = 'application/json'
  61. return params
  62. def _match_query(query):
  63. """
  64. The standard for full text queries.
  65. searx format: "key:value" e.g. city:berlin
  66. REF: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-query.html
  67. """
  68. try:
  69. key, value = query.split(':')
  70. except Exception as e:
  71. raise ValueError('query format must be "key:value"') from e
  72. return {"query": {"match": {key: {'query': value}}}}
  73. def _simple_query_string_query(query):
  74. """
  75. Accepts query strings, but it is less strict than query_string
  76. The field used can be specified in index.query.default_field in Elasticsearch.
  77. REF: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html
  78. """
  79. return {'query': {'simple_query_string': {'query': query}}}
  80. def _term_query(query):
  81. """
  82. Accepts one term and the name of the field.
  83. searx format: "key:value" e.g. city:berlin
  84. REF: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-term-query.html
  85. """
  86. try:
  87. key, value = query.split(':')
  88. except Exception as e:
  89. raise ValueError('query format must be key:value') from e
  90. return {'query': {'term': {key: value}}}
  91. def _terms_query(query):
  92. """
  93. Accepts multiple terms and the name of the field.
  94. searx format: "key:value1,value2" e.g. city:berlin,paris
  95. REF: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html
  96. """
  97. try:
  98. key, values = query.split(':')
  99. except Exception as e:
  100. raise ValueError('query format must be key:value1,value2') from e
  101. return {'query': {'terms': {key: values.split(',')}}}
  102. def _custom_query(query):
  103. key, value = query.split(':')
  104. custom_query = custom_query_json
  105. for query_key, query_value in custom_query.items():
  106. if query_key == '{{KEY}}':
  107. custom_query[key] = custom_query.pop(query_key)
  108. if query_value == '{{VALUE}}':
  109. custom_query[query_key] = value
  110. return custom_query
  111. def response(resp: SXNG_Response) -> EngineResults:
  112. res = EngineResults()
  113. resp_json = loads(resp.text)
  114. if 'error' in resp_json:
  115. raise SearxEngineAPIException(resp_json["error"])
  116. for result in resp_json["hits"]["hits"]:
  117. kvmap = {key: str(value) if not key.startswith("_") else value for key, value in result["_source"].items()}
  118. if show_metadata:
  119. kvmap["metadata"] = {"index": result["_index"], "id": result["_id"], "score": result["_score"]}
  120. res.add(res.types.KeyValue(kvmap=kvmap))
  121. return res
  122. _available_query_types = {
  123. # Full text queries
  124. # https://www.elastic.co/guide/en/elasticsearch/reference/current/full-text-queries.html
  125. 'match': _match_query,
  126. 'simple_query_string': _simple_query_string_query,
  127. # Term-level queries
  128. # https://www.elastic.co/guide/en/elasticsearch/reference/current/term-level-queries.html
  129. 'term': _term_query,
  130. 'terms': _terms_query,
  131. # Query JSON defined by the instance administrator.
  132. 'custom': _custom_query,
  133. }