[mod] migrate all key-value.html templates to KeyValue type

The engines now all use KeyValue results and return the results in a
EngineResults object.

The sqlite engine can return MainResult results in addition to KeyValue
results (based on engine's config in settings.yml),

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/command.py

@@ -81,6 +81,7 @@ from subprocess import Popen, PIPE
 from threading import Thread
 
 from searx import logger
+from searx.result_types import EngineResults
 
 
 engine_type = 'offline'
@@ -93,7 +94,6 @@ query_enum = []
 environment_variables = {}
 working_dir = realpath('.')
 result_separator = '\n'
-result_template = 'key-value.html'
 timeout = 4.0
 
 _command_logger = logger.getChild('command')
@@ -126,17 +126,17 @@ def init(engine_settings):
         environment_variables = engine_settings['environment_variables']
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
+    res = EngineResults()
     cmd = _get_command_to_run(query)
     if not cmd:
-        return []
+        return res
 
-    results = []
-    reader_thread = Thread(target=_get_results_from_process, args=(results, cmd, params['pageno']))
+    reader_thread = Thread(target=_get_results_from_process, args=(res, cmd, params['pageno']))
     reader_thread.start()
     reader_thread.join(timeout=timeout)
 
-    return results
+    return res
 
 
 def _get_command_to_run(query):
@@ -153,7 +153,7 @@ def _get_command_to_run(query):
     return cmd
 
 
-def _get_results_from_process(results, cmd, pageno):
+def _get_results_from_process(res: EngineResults, cmd, pageno):
     leftover = ''
     count = 0
     start, end = __get_results_limits(pageno)
@@ -173,12 +173,11 @@ def _get_results_from_process(results, cmd, pageno):
                     continue
 
                 if start <= count and count <= end:  # pylint: disable=chained-comparison
-                    result['template'] = result_template
-                    results.append(result)
+                    res.add(res.types.KeyValue(kvmap=result))
 
                 count += 1
                 if end < count:
-                    return results
+                    return res
 
             line = process.stdout.readline()
 

searx/engines/demo_offline.py

@@ -13,6 +13,7 @@ close to the implementation, its just a simple example.  To get in use of this
 """
 
 import json
+
 from searx.result_types import EngineResults
 
 engine_type = 'offline'
@@ -29,7 +30,7 @@ about = {
 }
 
 # if there is a need for globals, use a leading underline
-_my_offline_engine = None
+_my_offline_engine: str = ""
 
 
 def init(engine_settings=None):
@@ -50,24 +51,28 @@ def init(engine_settings=None):
 
 
 def search(query, request_params) -> EngineResults:
-    """Query (offline) engine and return results.  Assemble the list of results from
-    your local engine.  In this demo engine we ignore the 'query' term, usual
-    you would pass the 'query' term to your local engine to filter out the
+    """Query (offline) engine and return results.  Assemble the list of results
+    from your local engine.  In this demo engine we ignore the 'query' term,
+    usual you would pass the 'query' term to your local engine to filter out the
     results.
-
     """
     res = EngineResults()
 
-    result_list = json.loads(_my_offline_engine)
-
-    for row in result_list:
-        entry = {
+    count = 0
+    for row in json.loads(_my_offline_engine):
+        count += 1
+        kvmap = {
             'query': query,
             'language': request_params['searxng_locale'],
             'value': row.get("value"),
-            # choose a result template or comment out to use the *default*
-            'template': 'key-value.html',
         }
-        res.append(entry)
-
+        res.add(
+            res.types.KeyValue(
+                caption=f"Demo Offline Engine Result #{count}",
+                key_title="Name",
+                value_title="Value",
+                kvmap=kvmap,
+            )
+        )
+    res.add(res.types.LegacyResult(number_of_results=count))
     return res

searx/engines/elasticsearch.py

@@ -43,6 +43,8 @@ authentication configured to read from ``my-index`` index.
 
 from json import loads, dumps
 from searx.exceptions import SearxEngineAPIException
+from searx.result_types import EngineResults
+from searx.extended_types import SXNG_Response
 
 
 base_url = 'http://localhost:9200'
@@ -145,23 +147,20 @@ def _custom_query(query):
     return custom_query
 
 
-def response(resp):
-    results = []
+def response(resp: SXNG_Response) -> EngineResults:
+    res = EngineResults()
 
     resp_json = loads(resp.text)
     if 'error' in resp_json:
-        raise SearxEngineAPIException(resp_json['error'])
-
-    for result in resp_json['hits']['hits']:
-        r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()}
-        r['template'] = 'key-value.html'
+        raise SearxEngineAPIException(resp_json["error"])
 
+    for result in resp_json["hits"]["hits"]:
+        kvmap = {key: str(value) if not key.startswith("_") else value for key, value in result["_source"].items()}
         if show_metadata:
-            r['metadata'] = {'index': result['_index'], 'id': result['_id'], 'score': result['_score']}
-
-        results.append(r)
+            kvmap["metadata"] = {"index": result["_index"], "id": result["_id"], "score": result["_score"]}
+        res.add(res.types.KeyValue(kvmap=kvmap))
 
-    return results
+    return res
 
 
 _available_query_types = {

searx/engines/mariadb_server.py

@@ -35,6 +35,8 @@ except ImportError:
     # the engine
     pass
 
+from searx.result_types import EngineResults
+
 if TYPE_CHECKING:
     import logging
 
@@ -63,7 +65,6 @@ query_str = ""
 
 limit = 10
 paging = True
-result_template = 'key-value.html'
 _connection = None
 
 
@@ -79,17 +80,16 @@ def init(engine_settings):
     _connection = mariadb.connect(database=database, user=username, password=password, host=host, port=port)
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
     query_params = {'query': query}
     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
     logger.debug("SQL Query: %s", query_to_run)
+    res = EngineResults()
 
     with _connection.cursor() as cur:
         cur.execute(query_to_run, query_params)
-        results = []
         col_names = [i[0] for i in cur.description]
-        for res in cur:
-            result = dict(zip(col_names, map(str, res)))
-            result['template'] = result_template
-            results.append(result)
-        return results
+        for row in cur:
+            kvmap = dict(zip(col_names, map(str, row)))
+            res.add(res.types.KeyValue(kvmap=kvmap))
+    return res

searx/engines/meilisearch.py

@@ -33,15 +33,15 @@ Here is a simple example to query a Meilisearch instance:
 
 # pylint: disable=global-statement
 
-from json import loads, dumps
-
+from json import dumps
+from searx.result_types import EngineResults
+from searx.extended_types import SXNG_Response
 
 base_url = 'http://localhost:7700'
 index = ''
 auth_key = ''
 facet_filters = []
 _search_url = ''
-result_template = 'key-value.html'
 categories = ['general']
 paging = True
 
@@ -75,13 +75,12 @@ def request(query, params):
     return params
 
 
-def response(resp):
-    results = []
+def response(resp: SXNG_Response) -> EngineResults:
+    res = EngineResults()
 
-    resp_json = loads(resp.text)
-    for result in resp_json['hits']:
-        r = {key: str(value) for key, value in result.items()}
-        r['template'] = result_template
-        results.append(r)
+    resp_json = resp.json()
+    for row in resp_json['hits']:
+        kvmap = {key: str(value) for key, value in row.items()}
+        res.add(res.types.KeyValue(kvmap=kvmap))
 
-    return results
+    return res

searx/engines/mongodb.py

@@ -37,6 +37,7 @@ Implementations
 ===============
 
 """
+from __future__ import annotations
 
 import re
 
@@ -47,6 +48,8 @@ except ImportError:
     # to use the engine
     pass
 
+from searx.result_types import EngineResults
+
 
 engine_type = 'offline'
 
@@ -63,7 +66,6 @@ key = None
 paging = True
 results_per_page = 20
 exact_match_only = False
-result_template = 'key-value.html'
 
 _client = None
 
@@ -74,7 +76,7 @@ def init(_):
 
 def connect():
     global _client  # pylint: disable=global-statement
-    kwargs = {'port': port}
+    kwargs: dict[str, str | int] = {'port': port}
     if username:
         kwargs['username'] = username
     if password:
@@ -82,8 +84,8 @@ def connect():
     _client = MongoClient(host, **kwargs)[database][collection]
 
 
-def search(query, params):
-    results = []
+def search(query, params) -> EngineResults:
+    res = EngineResults()
     if exact_match_only:
         q = {'$eq': query}
     else:
@@ -92,11 +94,10 @@ def search(query, params):
 
     query = _client.find({key: q}).skip((params['pageno'] - 1) * results_per_page).limit(results_per_page)
 
-    results.append({'number_of_results': query.count()})
-    for r in query:
-        del r['_id']
-        r = {str(k): str(v) for k, v in r.items()}
-        r['template'] = result_template
-        results.append(r)
+    res.add(res.types.LegacyResult(number_of_results=query.count()))
+    for row in query:
+        del row['_id']
+        kvmap = {str(k): str(v) for k, v in row.items()}
+        res.add(res.types.KeyValue(kvmap=kvmap))
 
-    return results
+    return res

searx/engines/mysql_server.py

@@ -25,6 +25,8 @@ Implementations
 
 """
 
+from searx.result_types import EngineResults
+
 try:
     import mysql.connector  # type: ignore
 except ImportError:
@@ -55,7 +57,6 @@ query_str = ""
 
 limit = 10
 paging = True
-result_template = 'key-value.html'
 _connection = None
 
 
@@ -78,21 +79,15 @@ def init(engine_settings):
     )
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
+    res = EngineResults()
     query_params = {'query': query}
     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
 
     with _connection.cursor() as cur:
         cur.execute(query_to_run, query_params)
+        for row in cur:
+            kvmap = dict(zip(cur.column_names, map(str, row)))
+            res.add(res.types.KeyValue(kvmap=kvmap))
 
-        return _fetch_results(cur)
-
-
-def _fetch_results(cur):
-    results = []
-    for res in cur:
-        result = dict(zip(cur.column_names, map(str, res)))
-        result['template'] = result_template
-        results.append(result)
-
-    return results
+    return res

searx/engines/postgresql.py

@@ -28,6 +28,8 @@ except ImportError:
     # manually to use the engine.
     pass
 
+from searx.result_types import EngineResults
+
 engine_type = 'offline'
 
 host = "127.0.0.1"
@@ -50,7 +52,6 @@ query_str = ""
 
 limit = 10
 paging = True
-result_template = 'key-value.html'
 _connection = None
 
 
@@ -72,7 +73,7 @@ def init(engine_settings):
     )
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
     query_params = {'query': query}
     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
 
@@ -82,20 +83,16 @@ def search(query, params):
             return _fetch_results(cur)
 
 
-def _fetch_results(cur):
-    results = []
-    titles = []
-
+def _fetch_results(cur) -> EngineResults:
+    res = EngineResults()
     try:
         titles = [column_desc.name for column_desc in cur.description]
-
-        for res in cur:
-            result = dict(zip(titles, map(str, res)))
-            result['template'] = result_template
-            results.append(result)
+        for row in cur:
+            kvmap = dict(zip(titles, map(str, row)))
+            res.add(res.types.KeyValue(kvmap=kvmap))
 
     # no results to fetch
     except psycopg2.ProgrammingError:
         pass
 
-    return results
+    return res

searx/engines/redis_server.py

@@ -36,6 +36,8 @@ Implementations
 
 import redis  # pylint: disable=import-error
 
+from searx.result_types import EngineResults
+
 engine_type = 'offline'
 
 # redis connection variables
@@ -46,7 +48,6 @@ db = 0
 
 # engine specific variables
 paging = False
-result_template = 'key-value.html'
 exact_match_only = True
 
 _redis_client = None
@@ -63,30 +64,25 @@ def init(_engine_settings):
     )
 
 
-def search(query, _params):
+def search(query, _params) -> EngineResults:
+    res = EngineResults()
+
     if not exact_match_only:
-        return search_keys(query)
-
-    ret = _redis_client.hgetall(query)
-    if ret:
-        ret['template'] = result_template
-        return [ret]
-
-    if ' ' in query:
-        qset, rest = query.split(' ', 1)
-        ret = []
-        for res in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)):
-            ret.append(
-                {
-                    res[0]: res[1],
-                    'template': result_template,
-                }
-            )
-        return ret
-    return []
-
-
-def search_keys(query):
+        for kvmap in search_keys(query):
+            res.add(res.types.KeyValue(kvmap=kvmap))
+        return res
+
+    kvmap: dict[str, str] = _redis_client.hgetall(query)
+    if kvmap:
+        res.add(res.types.KeyValue(kvmap=kvmap))
+    elif " " in query:
+        qset, rest = query.split(" ", 1)
+        for row in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)):
+            res.add(res.types.KeyValue(kvmap={row[0]: row[1]}))
+    return res
+
+
+def search_keys(query) -> list[dict]:
     ret = []
     for key in _redis_client.scan_iter(match='*{}*'.format(query)):
         key_type = _redis_client.type(key)
@@ -98,7 +94,6 @@ def search_keys(query):
             res = dict(enumerate(_redis_client.lrange(key, 0, -1)))
 
         if res:
-            res['template'] = result_template
             res['redis_key'] = key
             ret.append(res)
     return ret

searx/engines/solr.py

@@ -29,9 +29,10 @@ This is an example configuration for searching in the collection
 
 # pylint: disable=global-statement
 
-from json import loads
 from urllib.parse import urlencode
 from searx.exceptions import SearxEngineAPIException
+from searx.result_types import EngineResults
+from searx.extended_types import SXNG_Response
 
 
 base_url = 'http://localhost:8983'
@@ -72,27 +73,21 @@ def request(query, params):
     return params
 
 
-def response(resp):
-    resp_json = __get_response(resp)
-
-    results = []
-    for result in resp_json['response']['docs']:
-        r = {key: str(value) for key, value in result.items()}
-        if len(r) == 0:
-            continue
-        r['template'] = 'key-value.html'
-        results.append(r)
-
-    return results
-
-
-def __get_response(resp):
+def response(resp: SXNG_Response) -> EngineResults:
     try:
-        resp_json = loads(resp.text)
+        resp_json = resp.json()
     except Exception as e:
         raise SearxEngineAPIException("failed to parse response") from e
 
-    if 'error' in resp_json:
-        raise SearxEngineAPIException(resp_json['error']['msg'])
+    if "error" in resp_json:
+        raise SearxEngineAPIException(resp_json["error"]["msg"])
+
+    res = EngineResults()
+
+    for result in resp_json["response"]["docs"]:
+        kvmap = {key: str(value) for key, value in result.items()}
+        if not kvmap:
+            continue
+        res.add(res.types.KeyValue(kvmap=kvmap))
 
-    return resp_json
+    return res

searx/engines/sqlite.py

@@ -2,6 +2,14 @@
 """SQLite is a small, fast and reliable SQL database engine.  It does not require
 any extra dependency.
 
+Configuration
+=============
+
+The engine has the following (additional) settings:
+
+- :py:obj:`result_type`
+
+
 Example
 =======
 
@@ -18,29 +26,32 @@ Query to test: ``!mediathekview concert``
 
 .. code:: yaml
 
-   - name: mediathekview
-     engine: sqlite
-     disabled: False
-     categories: general
-     result_template: default.html
-     database: searx/data/filmliste-v2.db
-     query_str:  >-
-       SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
-              COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
-              description AS content
-         FROM film
-        WHERE title LIKE :wildcard OR description LIKE :wildcard
-        ORDER BY duration DESC
+  - name: mediathekview
+    engine: sqlite
+    shortcut: mediathekview
+    categories: [general, videos]
+    result_type: MainResult
+    database: searx/data/filmliste-v2.db
+    query_str: >-
+      SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
+             COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
+             description AS content
+        FROM film
+       WHERE title LIKE :wildcard OR description LIKE :wildcard
+       ORDER BY duration DESC
 
 Implementations
 ===============
 
 """
-
+import typing
 import sqlite3
 import contextlib
 
-engine_type = 'offline'
+from searx.result_types import EngineResults
+from searx.result_types import MainResult, KeyValue
+
+engine_type = "offline"
 
 database = ""
 """Filename of the SQLite DB."""
@@ -48,9 +59,11 @@ database = ""
 query_str = ""
 """SQL query that returns the result items."""
 
+result_type: typing.Literal["MainResult", "KeyValue"] = "KeyValue"
+"""The result type can be :py:obj:`MainResult` or :py:obj:`KeyValue`."""
+
 limit = 10
 paging = True
-result_template = 'key-value.html'
 
 
 def init(engine_settings):
@@ -80,9 +93,8 @@ def sqlite_cursor():
             yield cursor
 
 
-def search(query, params):
-    results = []
-
+def search(query, params) -> EngineResults:
+    res = EngineResults()
     query_params = {
         'query': query,
         'wildcard': r'%' + query.replace(' ', r'%') + r'%',
@@ -97,9 +109,11 @@ def search(query, params):
         col_names = [cn[0] for cn in cur.description]
 
         for row in cur.fetchall():
-            item = dict(zip(col_names, map(str, row)))
-            item['template'] = result_template
-            logger.debug("append result --> %s", item)
-            results.append(item)
-
-    return results
+            kvmap = dict(zip(col_names, map(str, row)))
+            if result_type == "MainResult":
+                item = MainResult(**kvmap)  # type: ignore
+            else:
+                item = KeyValue(kvmap=kvmap)
+            res.add(item)
+
+    return res

searx/settings.yml

@@ -1888,15 +1888,15 @@ engines:
   # For this demo of the sqlite engine download:
   #   https://liste.mediathekview.de/filmliste-v2.db.bz2
   # and unpack into searx/data/filmliste-v2.db
-  # Query to test: "!demo concert"
+  # Query to test: "!mediathekview concert"
   #
-  # - name: demo
+  # - name: mediathekview
   #   engine: sqlite
-  #   shortcut: demo
-  #   categories: general
-  #   result_template: default.html
+  #   shortcut: mediathekview
+  #   categories: [general, videos]
+  #   result_type: MainResult
   #   database: searx/data/filmliste-v2.db
-  #   query_str:  >-
+  #   query_str: >-
   #     SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
   #            COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
   #            description AS content

searx/templates/simple/result_templates/key-value.html

@@ -1,11 +0,0 @@
-<table>
-    {% for key, value in result.items() %}
-    {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions', 'parsed_url'] %}
-        {% continue %}
-    {% endif %}
-    <tr>
-        <td><b>{{ key|upper }}</b>: {{ value }}</td>
-    </tr>
-    {% endfor %}
-</table>
-<div class="engines">{% for engine in result.engines %}<span>{{ engine }}</span>{% endfor %}</div>{{- '' -}}