[lint] pylint searx/search/processors files / BTW add some doc-strings

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/search/processors/__init__.py

@@ -1,37 +1,49 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+
+"""Implement request processores used by engine-types.
+
+"""
+
+__all__ = [
+    'EngineProcessor',
+    'OfflineProcessor',
+    'OnlineProcessor',
+    'OnlineDictionaryProcessor',
+    'OnlineCurrencyProcessor',
+    'processors',
+]
+
+from searx import logger
+import searx.engines as engines
 
 from .online import OnlineProcessor
 from .offline import OfflineProcessor
 from .online_dictionary import OnlineDictionaryProcessor
 from .online_currency import OnlineCurrencyProcessor
 from .abstract import EngineProcessor
-from searx import logger
-import searx.engines as engines
-
 
-__all__ = ['EngineProcessor', 'OfflineProcessor', 'OnlineProcessor',
-           'OnlineDictionaryProcessor', 'OnlineCurrencyProcessor', 'processors']
 logger = logger.getChild('search.processors')
 processors = {}
-
+"""Cache request processores, stored by *engine-name* (:py:func:`initialize`)"""
 
 def get_processor_class(engine_type):
+    """Return processor class according to the ``engine_type``"""
     for c in [OnlineProcessor, OfflineProcessor, OnlineDictionaryProcessor, OnlineCurrencyProcessor]:
         if c.engine_type == engine_type:
             return c
     return None
 
-
 def get_processor(engine, engine_name):
+    """Return processor instance that fits to ``engine.engine.type``)"""
     engine_type = getattr(engine, 'engine_type', 'online')
     processor_class = get_processor_class(engine_type)
     if processor_class:
         return processor_class(engine, engine_name)
-    else:
-        return None
-
+    return None
 
 def initialize(engine_list):
+    """Initialize all engines and store a processor for each engine in :py:obj:`processors`."""
     engines.initialize_engines(engine_list)
     for engine_name, engine in engines.engines.items():
         processor = get_processor(engine, engine_name)

searx/search/processors/abstract.py

@@ -1,4 +1,9 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+
+"""Abstract base classes for engine request processores.
+
+"""
 
 import threading
 from abc import abstractmethod, ABC
@@ -10,12 +15,13 @@ from searx.network import get_time_for_thread, get_network
 from searx.metrics import histogram_observe, counter_inc, count_exception, count_error
 from searx.exceptions import SearxEngineAccessDeniedException
 
-
 logger = logger.getChild('searx.search.processor')
 SUSPENDED_STATUS = {}
 
+# pylint: disable=missing-function-docstring
 
 class SuspendedStatus:
+    """Class to handle suspend state."""
 
     __slots__ = 'suspend_end_time', 'suspend_reason', 'continuous_errors', 'lock'
 
@@ -49,6 +55,7 @@ class SuspendedStatus:
 
 
 class EngineProcessor(ABC):
+    """Base classes used for all types of reqest processores."""
 
     __slots__ = 'engine', 'engine_name', 'lock', 'suspended_status'
 
@@ -143,9 +150,7 @@ class EngineProcessor(ABC):
         if tests is None:
             tests = getattr(self.engine, 'additional_tests', {})
             tests.update(self.get_default_tests())
-            return tests
-        else:
-            return tests
+        return tests
 
-    def get_default_tests(self):
+    def get_default_tests(self):  # pylint: disable=no-self-use
         return {}

searx/search/processors/offline.py

@@ -1,13 +1,17 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
 
-from searx import logger
-from searx.search.processors.abstract import EngineProcessor
+"""Processores for engine-type: ``offline``
 
+"""
 
-logger = logger.getChild('searx.search.processor.offline')
+from searx import logger
+from .abstract import EngineProcessor
 
+logger = logger.getChild('searx.search.processor.offline')
 
 class OfflineProcessor(EngineProcessor):
+    """Processor class used by ``offline`` engines"""
 
     engine_type = 'offline'
 
@@ -21,6 +25,6 @@ class OfflineProcessor(EngineProcessor):
         except ValueError as e:
             # do not record the error
             logger.exception('engine {0} : invalid input : {1}'.format(self.engine_name, e))
-        except Exception as e:
+        except Exception as e: # pylint: disable=broad-except
             self.handle_exception(result_container, e)
             logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e))

searx/search/processors/online.py

@@ -1,24 +1,29 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+
+"""Processores for engine-type: ``online``
+
+"""
 
 from time import time
 import asyncio
-
 import httpx
 
 import searx.network
 from searx import logger
 from searx.utils import gen_useragent
-from searx.exceptions import (SearxEngineAccessDeniedException, SearxEngineCaptchaException,
-                              SearxEngineTooManyRequestsException,)
+from searx.exceptions import (
+    SearxEngineAccessDeniedException,
+    SearxEngineCaptchaException,
+    SearxEngineTooManyRequestsException,
+)
 from searx.metrics.error_recorder import count_error
-
-from searx.search.processors.abstract import EngineProcessor
-
+from .abstract import EngineProcessor
 
 logger = logger.getChild('searx.search.processor.online')
 
-
 def default_request_params():
+    """Default request parameters for ``online`` engines."""
     return {
         'method': 'GET',
         'headers': {},
@@ -31,6 +36,7 @@ def default_request_params():
 
 
 class OnlineProcessor(EngineProcessor):
+    """Processor class for ``online`` engines."""
 
     engine_type = 'online'
 
@@ -153,7 +159,7 @@ class OnlineProcessor(EngineProcessor):
         except SearxEngineAccessDeniedException as e:
             self.handle_exception(result_container, e, suspend=True)
             logger.exception('engine {0} : Searx is blocked'.format(self.engine_name))
-        except Exception as e:
+        except Exception as e:  # pylint: disable=broad-except
             self.handle_exception(result_container, e)
             logger.exception('engine {0} : exception : {1}'.format(self.engine_name, e))
 

searx/search/processors/online_currency.py

@@ -1,4 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Processores for engine-type: ``online_currency``
+
+"""
 
 import unicodedata
 import re
@@ -6,32 +10,31 @@ import re
 from searx.data import CURRENCIES
 from .online import OnlineProcessor
 
-
 parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
 
+# pylint: disable=missing-function-docstring
 
 def normalize_name(name):
     name = name.lower().replace('-', ' ').rstrip('s')
     name = re.sub(' +', ' ', name)
     return unicodedata.normalize('NFKD', name).lower()
 
-
 def name_to_iso4217(name):
-    global CURRENCIES
+    global CURRENCIES  # pylint: disable=global-statement
     name = normalize_name(name)
     currency = CURRENCIES['names'].get(name, [name])
     if isinstance(currency, str):
         return currency
     return currency[0]
 
-
 def iso4217_to_name(iso4217, language):
-    global CURRENCIES
+    global CURRENCIES  # pylint: disable=global-statement
     return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)
 
-
 class OnlineCurrencyProcessor(OnlineProcessor):
 
+    """Processor class used by ``online_currency`` engines."""
+
     engine_type = 'online_currency'
 
     def get_params(self, search_query, engine_category):

searx/search/processors/online_dictionary.py

@@ -1,15 +1,18 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Processores for engine-type: ``online_dictionary``
+
+"""
 
 import re
 
 from searx.utils import is_valid_lang
 from .online import OnlineProcessor
 
-
 parser_re = re.compile('.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
 
-
 class OnlineDictionaryProcessor(OnlineProcessor):
+    """Processor class used by ``online_dictionnary`` engines."""
 
     engine_type = 'online_dictionnary'