[refactor] add type hints & remove Setting._post_init

Previously the Setting classes used a horrible _post_init
hack that prevented proper type checking.

searx/engines/__init__.py

@@ -13,7 +13,7 @@ usage::
 
 import sys
 import copy
-from typing import List
+from typing import Dict, List, Optional
 
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
@@ -67,10 +67,10 @@ class Engine:  # pylint: disable=too-few-public-methods
     timeout: float
 
 
-# Defaults for the namespace of an engine module, see :py:func:`load_engine``
+# Defaults for the namespace of an engine module, see :py:func:`load_engine`
 
 categories = {'general': []}
-engines = {}
+engines: Dict[str, Engine] = {}
 engine_shortcuts = {}
 """Simple map of registered *shortcuts* to name of the engine (or ``None``).
 
@@ -81,7 +81,7 @@ engine_shortcuts = {}
 """
 
 
-def load_engine(engine_data):
+def load_engine(engine_data: dict) -> Optional[Engine]:
     """Load engine from ``engine_data``.
 
     :param dict engine_data:  Attributes from YAML ``settings:engines/<engine>``
@@ -157,7 +157,7 @@ def set_loggers(engine, engine_name):
             module.logger = logger.getChild(module_engine_name)
 
 
-def update_engine_attributes(engine, engine_data):
+def update_engine_attributes(engine: Engine, engine_data):
     # set engine attributes from engine_data
     for param_name, param_value in engine_data.items():
         if param_name == 'categories':
@@ -175,7 +175,7 @@ def update_engine_attributes(engine, engine_data):
             setattr(engine, arg_name, copy.deepcopy(arg_value))
 
 
-def set_language_attributes(engine):
+def set_language_attributes(engine: Engine):
     # assign supported languages from json file
     if engine.name in ENGINES_LANGUAGES:
         engine.supported_languages = ENGINES_LANGUAGES[engine.name]
@@ -248,7 +248,7 @@ def is_missing_required_attributes(engine):
     return missing
 
 
-def is_engine_active(engine):
+def is_engine_active(engine: Engine):
     # check if engine is inactive
     if engine.inactive is True:
         return False
@@ -260,7 +260,7 @@ def is_engine_active(engine):
     return True
 
 
-def register_engine(engine):
+def register_engine(engine: Engine):
     if engine.name in engines:
         logger.error('Engine config error: ambigious name: {0}'.format(engine.name))
         sys.exit(1)

searx/plugins/__init__.py

@@ -10,10 +10,20 @@ from os.path import abspath, basename, dirname, exists, join
 from shutil import copyfile
 from pkgutil import iter_modules
 from logging import getLogger
+from typing import List
 
 from searx import logger, settings
 
 
+class Plugin:  # pylint: disable=too-few-public-methods
+    """This class is currently never initialized and only used for type hinting."""
+
+    id: str
+    name: str
+    description: str
+    default_on: bool
+
+
 logger = logger.getChild("plugins")
 
 required_attrs = (
@@ -175,7 +185,7 @@ def load_and_initialize_plugin(plugin_module_name, external, init_args):
 
 class PluginStore:
     def __init__(self):
-        self.plugins = []
+        self.plugins: List[Plugin] = []
 
     def __iter__(self):
         for plugin in self.plugins:

searx/preferences.py

@@ -8,8 +8,14 @@
 from base64 import urlsafe_b64encode, urlsafe_b64decode
 from zlib import compress, decompress
 from urllib.parse import parse_qs, urlencode
+from typing import Iterable, Dict, List, Set
+from dataclasses import dataclass
+
+import flask
 
 from searx import settings, autocomplete
+from searx.engines import Engine
+from searx.plugins import Plugin
 from searx.locales import LOCALE_NAMES
 from searx.webutils import VALID_LANGUAGE_CODE
 from searx.engines import OTHER_CATEGORY
@@ -21,31 +27,20 @@ ENABLED = 1
 DOI_RESOLVERS = list(settings['doi_resolvers'])
 
 
-class MissingArgumentException(Exception):
-    """Exption from ``cls._post_init`` when a argument is missed."""
-
-
 class ValidationException(Exception):
 
-    """Exption from ``cls._post_init`` when configuration value is invalid."""
+    """Exption from ``cls.__init__`` when configuration value is invalid."""
 
 
 class Setting:
     """Base class of user settings"""
 
-    def __init__(self, default_value, locked=False, **kwargs):
+    def __init__(self, default_value, locked: bool = False):
         super().__init__()
         self.value = default_value
         self.locked = locked
-        for key, value in kwargs.items():
-            setattr(self, key, value)
 
-        self._post_init()
-
-    def _post_init(self):
-        pass
-
-    def parse(self, data):
+    def parse(self, data: str):
         """Parse ``data`` and store the result at ``self.value``
 
         If needed, its overwritten in the inheritance.
@@ -59,7 +54,7 @@ class Setting:
         """
         return self.value
 
-    def save(self, name, resp):
+    def save(self, name: str, resp: flask.Response):
         """Save cookie ``name`` in the HTTP reponse obect
 
         If needed, its overwritten in the inheritance."""
@@ -73,35 +68,35 @@ class StringSetting(Setting):
 class EnumStringSetting(Setting):
     """Setting of a value which can only come from the given choices"""
 
-    def _post_init(self):
-        if not hasattr(self, 'choices'):
-            raise MissingArgumentException('Missing argument: choices')
+    def __init__(self, default_value: str, choices: Iterable[str], locked=False):
+        super().__init__(default_value, locked)
+        self.choices = choices
         self._validate_selection(self.value)
 
-    def _validate_selection(self, selection):
-        if selection not in self.choices:  # pylint: disable=no-member
+    def _validate_selection(self, selection: str):
+        if selection not in self.choices:
             raise ValidationException('Invalid value: "{0}"'.format(selection))
 
-    def parse(self, data):
+    def parse(self, data: str):
         """Parse and validate ``data`` and store the result at ``self.value``"""
         self._validate_selection(data)
         self.value = data
 
 
-class MultipleChoiceSetting(EnumStringSetting):
+class MultipleChoiceSetting(Setting):
     """Setting of values which can only come from the given choices"""
 
-    def _validate_selections(self, selections):
+    def __init__(self, default_value: List[str], choices: Iterable[str], locked=False):
+        super().__init__(default_value, locked)
+        self.choices = choices
+        self._validate_selections(self.value)
+
+    def _validate_selections(self, selections: List[str]):
         for item in selections:
-            if item not in self.choices:  # pylint: disable=no-member
+            if item not in self.choices:
                 raise ValidationException('Invalid value: "{0}"'.format(selections))
 
-    def _post_init(self):
-        if not hasattr(self, 'choices'):
-            raise MissingArgumentException('Missing argument: choices')
-        self._validate_selections(self.value)
-
-    def parse(self, data):
+    def parse(self, data: str):
         """Parse and validate ``data`` and store the result at ``self.value``"""
         if data == '':
             self.value = []
@@ -111,16 +106,16 @@ class MultipleChoiceSetting(EnumStringSetting):
         self._validate_selections(elements)
         self.value = elements
 
-    def parse_form(self, data):
+    def parse_form(self, data: List[str]):
         if self.locked:
             return
 
         self.value = []
         for choice in data:
-            if choice in self.choices and choice not in self.value:  # pylint: disable=no-member
+            if choice in self.choices and choice not in self.value:
                 self.value.append(choice)
 
-    def save(self, name, resp):
+    def save(self, name: str, resp: flask.Response):
         """Save cookie ``name`` in the HTTP reponse obect"""
         resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE)
 
@@ -128,32 +123,32 @@ class MultipleChoiceSetting(EnumStringSetting):
 class SetSetting(Setting):
     """Setting of values of type ``set`` (comma separated string)"""
 
-    def _post_init(self):
-        if not hasattr(self, 'values'):
-            self.values = set()
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.values = set()
 
     def get_value(self):
         """Returns a string with comma separated values."""
         return ','.join(self.values)
 
-    def parse(self, data):
+    def parse(self, data: str):
         """Parse and validate ``data`` and store the result at ``self.value``"""
         if data == '':
-            self.values = set()  # pylint: disable=attribute-defined-outside-init
+            self.values = set()
             return
 
         elements = data.split(',')
         for element in elements:
             self.values.add(element)
 
-    def parse_form(self, data):
+    def parse_form(self, data: str):
         if self.locked:
             return
 
         elements = data.split(',')
-        self.values = set(elements)  # pylint: disable=attribute-defined-outside-init
+        self.values = set(elements)
 
-    def save(self, name, resp):
+    def save(self, name: str, resp: flask.Response):
         """Save cookie ``name`` in the HTTP reponse obect"""
         resp.set_cookie(name, ','.join(self.values), max_age=COOKIE_MAX_AGE)
 
@@ -165,13 +160,13 @@ class SearchLanguageSetting(EnumStringSetting):
         if selection != '' and not VALID_LANGUAGE_CODE.match(selection):
             raise ValidationException('Invalid language code: "{0}"'.format(selection))
 
-    def parse(self, data):
+    def parse(self, data: str):
         """Parse and validate ``data`` and store the result at ``self.value``"""
-        if data not in self.choices and data != self.value:  # pylint: disable=no-member
+        if data not in self.choices and data != self.value:
             # hack to give some backwards compatibility with old language cookies
             data = str(data).replace('_', '-')
             lang = data.split('-', maxsplit=1)[0]
-            # pylint: disable=no-member
+
             if data in self.choices:
                 pass
             elif lang in self.choices:
@@ -185,34 +180,43 @@ class SearchLanguageSetting(EnumStringSetting):
 class MapSetting(Setting):
     """Setting of a value that has to be translated in order to be storable"""
 
-    def _post_init(self):
-        if not hasattr(self, 'map'):
-            raise MissingArgumentException('missing argument: map')
-        if self.value not in self.map.values():  # pylint: disable=no-member
+    def __init__(self, default_value, map: Dict[str, object], locked=False):  # pylint: disable=redefined-builtin
+        super().__init__(default_value, locked)
+        self.map = map
+
+        if self.value not in self.map.values():
             raise ValidationException('Invalid default value')
 
-    def parse(self, data):
+    def parse(self, data: str):
         """Parse and validate ``data`` and store the result at ``self.value``"""
-        # pylint: disable=no-member
+
         if data not in self.map:
             raise ValidationException('Invalid choice: {0}'.format(data))
         self.value = self.map[data]
         self.key = data  # pylint: disable=attribute-defined-outside-init
 
-    def save(self, name, resp):
+    def save(self, name: str, resp: flask.Response):
         """Save cookie ``name`` in the HTTP reponse obect"""
         if hasattr(self, 'key'):
             resp.set_cookie(name, self.key, max_age=COOKIE_MAX_AGE)
 
 
+@dataclass
+class Choice:
+    """A choice for a ``SwitchableSetting``."""
+
+    default_on: bool
+    id: str
+
+
 class SwitchableSetting(Setting):
     """Base class for settings that can be turned on && off"""
 
-    def _post_init(self):
-        self.disabled = set()
-        self.enabled = set()
-        if not hasattr(self, 'choices'):
-            raise MissingArgumentException('missing argument: choices')
+    def __init__(self, default_value, locked: bool, choices: Iterable[Choice]):
+        super().__init__(default_value, locked)
+        self.choices = choices
+        self.enabled: Set[str] = set()
+        self.disabled: Set[str] = set()
 
     def transform_form_items(self, items):
         # pylint: disable=no-self-use
@@ -223,62 +227,57 @@ class SwitchableSetting(Setting):
         return values
 
     def parse_cookie(self, data):
-        # pylint: disable=attribute-defined-outside-init
         if data[DISABLED] != '':
             self.disabled = set(data[DISABLED].split(','))
         if data[ENABLED] != '':
             self.enabled = set(data[ENABLED].split(','))
 
-    def parse_form(self, items):
+    def parse_form(self, items: List[str]):
         if self.locked:
             return
 
         items = self.transform_form_items(items)
-        self.disabled = set()  # pylint: disable=attribute-defined-outside-init
-        self.enabled = set()  # pylint: disable=attribute-defined-outside-init
-        for choice in self.choices:  # pylint: disable=no-member
-            if choice['default_on']:
-                if choice['id'] in items:
-                    self.disabled.add(choice['id'])
+        self.disabled = set()
+        self.enabled = set()
+        for choice in self.choices:
+            if choice.default_on:
+                if choice.id in items:
+                    self.disabled.add(choice.id)
             else:
-                if choice['id'] not in items:
-                    self.enabled.add(choice['id'])
+                if choice.id not in items:
+                    self.enabled.add(choice.id)
 
-    def save(self, resp):  # pylint: disable=arguments-differ
+    def save(self, resp: flask.Response):  # pylint: disable=arguments-differ
         """Save cookie in the HTTP reponse obect"""
         resp.set_cookie('disabled_{0}'.format(self.value), ','.join(self.disabled), max_age=COOKIE_MAX_AGE)
         resp.set_cookie('enabled_{0}'.format(self.value), ','.join(self.enabled), max_age=COOKIE_MAX_AGE)
 
     def get_disabled(self):
         disabled = self.disabled
-        for choice in self.choices:  # pylint: disable=no-member
-            if not choice['default_on'] and choice['id'] not in self.enabled:
-                disabled.add(choice['id'])
+        for choice in self.choices:
+            if not choice.default_on and choice.id not in self.enabled:
+                disabled.add(choice.id)
         return self.transform_values(disabled)
 
     def get_enabled(self):
         enabled = self.enabled
-        for choice in self.choices:  # pylint: disable=no-member
-            if choice['default_on'] and choice['id'] not in self.disabled:
-                enabled.add(choice['id'])
+        for choice in self.choices:
+            if choice.default_on and choice.id not in self.disabled:
+                enabled.add(choice.id)
         return self.transform_values(enabled)
 
 
 class EnginesSetting(SwitchableSetting):
     """Engine settings"""
 
-    def _post_init(self):
-        super()._post_init()
-        transformed_choices = []
-        for engine_name, engine in self.choices.items():  # pylint: disable=no-member,access-member-before-definition
+    def __init__(self, default_value, engines: Iterable[Engine]):
+        choices = []
+        for engine in engines:
             for category in engine.categories:
                 if not category in list(settings['categories_as_tabs'].keys()) + [OTHER_CATEGORY]:
                     continue
-                transformed_choice = {}
-                transformed_choice['default_on'] = not engine.disabled
-                transformed_choice['id'] = '{}__{}'.format(engine_name, category)
-                transformed_choices.append(transformed_choice)
-        self.choices = transformed_choices
+                choices.append(Choice(default_on=not engine.disabled, id='{}__{}'.format(engine.name, category)))
+        super().__init__(default_value, False, choices)
 
     def transform_form_items(self, items):
         return [item[len('engine_') :].replace('_', ' ').replace('  ', '__') for item in items]
@@ -296,15 +295,11 @@ class EnginesSetting(SwitchableSetting):
 class PluginsSetting(SwitchableSetting):
     """Plugin settings"""
 
-    def _post_init(self):
-        super()._post_init()
-        transformed_choices = []
-        for plugin in self.choices:  # pylint: disable=access-member-before-definition
-            transformed_choice = {}
-            transformed_choice['default_on'] = plugin.default_on
-            transformed_choice['id'] = plugin.id
-            transformed_choices.append(transformed_choice)
-        self.choices = transformed_choices
+    def __init__(self, default_value, plugins: Iterable[Plugin]):
+        choices = []
+        for plugin in plugins:
+            choices.append(Choice(default_on=plugin.default_on, id=plugin.id))
+        super().__init__(default_value, False, choices)
 
     def transform_form_items(self, items):
         return [item[len('plugin_') :] for item in items]
@@ -313,34 +308,34 @@ class PluginsSetting(SwitchableSetting):
 class Preferences:
     """Validates and saves preferences to cookies"""
 
-    def __init__(self, themes, categories, engines, plugins):
+    def __init__(self, themes: List[str], categories: List[str], engines: Dict[str, Engine], plugins: Iterable[Plugin]):
         super().__init__()
 
-        self.key_value_settings = {
+        self.key_value_settings: Dict[str, Setting] = {
             # fmt: off
             'categories': MultipleChoiceSetting(
                 ['general'],
-                is_locked('categories'),
+                locked=is_locked('categories'),
                 choices=categories + ['none']
             ),
             'language': SearchLanguageSetting(
                 settings['search']['default_lang'],
-                is_locked('language'),
+                locked=is_locked('language'),
                 choices=settings['search']['languages'] + ['']
             ),
             'locale': EnumStringSetting(
                 settings['ui']['default_locale'],
-                is_locked('locale'),
+                locked=is_locked('locale'),
                 choices=list(LOCALE_NAMES.keys()) + ['']
             ),
             'autocomplete': EnumStringSetting(
                 settings['search']['autocomplete'],
-                is_locked('autocomplete'),
+                locked=is_locked('autocomplete'),
                 choices=list(autocomplete.backends.keys()) + ['']
             ),
             'image_proxy': MapSetting(
                 settings['server']['image_proxy'],
-                is_locked('image_proxy'),
+                locked=is_locked('image_proxy'),
                 map={
                     '': settings['server']['image_proxy'],
                     '0': False,
@@ -351,12 +346,12 @@ class Preferences:
             ),
             'method': EnumStringSetting(
                 settings['server']['method'],
-                is_locked('method'),
+                locked=is_locked('method'),
                 choices=('GET', 'POST')
             ),
             'safesearch': MapSetting(
                 settings['search']['safe_search'],
-                is_locked('safesearch'),
+                locked=is_locked('safesearch'),
                 map={
                     '0': 0,
                     '1': 1,
@@ -365,12 +360,12 @@ class Preferences:
             ),
             'theme': EnumStringSetting(
                 settings['ui']['default_theme'],
-                is_locked('theme'),
+                locked=is_locked('theme'),
                 choices=themes
             ),
             'results_on_new_tab': MapSetting(
                 settings['ui']['results_on_new_tab'],
-                is_locked('results_on_new_tab'),
+                locked=is_locked('results_on_new_tab'),
                 map={
                     '0': False,
                     '1': True,
@@ -380,22 +375,22 @@ class Preferences:
             ),
             'doi_resolver': MultipleChoiceSetting(
                 [settings['default_doi_resolver'], ],
-                is_locked('doi_resolver'),
+                locked=is_locked('doi_resolver'),
                 choices=DOI_RESOLVERS
             ),
             'oscar-style': EnumStringSetting(
                 settings['ui']['theme_args']['oscar_style'],
-                is_locked('oscar-style'),
+                locked=is_locked('oscar-style'),
                 choices=['', 'logicodev', 'logicodev-dark', 'pointhi']
             ),
             'simple_style': EnumStringSetting(
                 settings['ui']['theme_args']['simple_style'],
-                is_locked('simple_style'),
+                locked=is_locked('simple_style'),
                 choices=['', 'auto', 'light', 'dark']
             ),
             'advanced_search': MapSetting(
                 settings['ui']['advanced_search'],
-                is_locked('advanced_search'),
+                locked=is_locked('advanced_search'),
                 map={
                     '0': False,
                     '1': True,
@@ -406,7 +401,7 @@ class Preferences:
             ),
             'query_in_title': MapSetting(
                 settings['ui']['query_in_title'],
-                is_locked('query_in_title'),
+                locked=is_locked('query_in_title'),
                 map={
                     '': settings['ui']['query_in_title'],
                     '0': False,
@@ -418,10 +413,10 @@ class Preferences:
             # fmt: on
         }
 
-        self.engines = EnginesSetting('engines', choices=engines)
-        self.plugins = PluginsSetting('plugins', choices=plugins)
+        self.engines = EnginesSetting('engines', engines=engines.values())
+        self.plugins = PluginsSetting('plugins', plugins=plugins)
         self.tokens = SetSetting('tokens')
-        self.unknown_params = {}
+        self.unknown_params: Dict[str, str] = {}
 
     def get_as_url_params(self):
         """Return preferences as URL parameters"""
@@ -444,7 +439,7 @@ class Preferences:
 
         return urlsafe_b64encode(compress(urlencode(settings_kv).encode())).decode()
 
-    def parse_encoded_data(self, input_data):
+    def parse_encoded_data(self, input_data: str):
         """parse (base64) preferences from request (``flask.request.form['preferences']``)"""
         bin_data = decompress(urlsafe_b64decode(input_data))
         dict_data = {}
@@ -452,7 +447,7 @@ class Preferences:
             dict_data[x] = y[0]
         self.parse_dict(dict_data)
 
-    def parse_dict(self, input_data):
+    def parse_dict(self, input_data: Dict[str, str]):
         """parse preferences from request (``flask.request.form``)"""
         for user_setting_name, user_setting in input_data.items():
             if user_setting_name in self.key_value_settings:
@@ -474,7 +469,7 @@ class Preferences:
             ):
                 self.unknown_params[user_setting_name] = user_setting
 
-    def parse_form(self, input_data):
+    def parse_form(self, input_data: Dict[str, str]):
         """Parse formular (``<input>``) data from a ``flask.request.form``"""
         disabled_engines = []
         enabled_categories = []
@@ -497,7 +492,7 @@ class Preferences:
         self.plugins.parse_form(disabled_plugins)
 
     # cannot be used in case of engines or plugins
-    def get_value(self, user_setting_name):
+    def get_value(self, user_setting_name: str):
         """Returns the value for ``user_setting_name``"""
         ret_val = None
         if user_setting_name in self.key_value_settings:
@@ -506,7 +501,7 @@ class Preferences:
             ret_val = self.unknown_params[user_setting_name]
         return ret_val
 
-    def save(self, resp):
+    def save(self, resp: flask.Response):
         """Save cookie in the HTTP reponse obect"""
         for user_setting_name, user_setting in self.key_value_settings.items():
             # pylint: disable=unnecessary-dict-index-lookup
@@ -532,7 +527,7 @@ class Preferences:
         return valid
 
 
-def is_locked(setting_name):
+def is_locked(setting_name: str):
     """Checks if a given setting name is locked by settings.yml"""
     if 'preferences' not in settings:
         return False

tests/unit/test_preferences.py

@@ -1,7 +1,6 @@
 from searx.preferences import (
     EnumStringSetting,
     MapSetting,
-    MissingArgumentException,
     SearchLanguageSetting,
     MultipleChoiceSetting,
     PluginsSetting,
@@ -19,10 +18,6 @@ class PluginStub:
 class TestSettings(SearxTestCase):
     # map settings
 
-    def test_map_setting_invalid_initialization(self):
-        with self.assertRaises(MissingArgumentException):
-            MapSetting(3, wrong_argument={'0': 0})
-
     def test_map_setting_invalid_default_value(self):
         with self.assertRaises(ValidationException):
             MapSetting(3, map={'dog': 1, 'bat': 2})
@@ -43,9 +38,6 @@ class TestSettings(SearxTestCase):
         self.assertEqual(setting.get_value(), 2)
 
     # enum settings
-    def test_enum_setting_invalid_initialization(self):
-        with self.assertRaises(MissingArgumentException):
-            EnumStringSetting('cat', wrong_argument=[0, 1, 2])
 
     def test_enum_setting_invalid_default_value(self):
         with self.assertRaises(ValidationException):
@@ -67,9 +59,6 @@ class TestSettings(SearxTestCase):
         self.assertEqual(setting.get_value(), 2)
 
     # multiple choice settings
-    def test_multiple_setting_invalid_initialization(self):
-        with self.assertRaises(MissingArgumentException):
-            MultipleChoiceSetting(['2'], wrong_argument=['0', '1', '2'])
 
     def test_multiple_setting_invalid_default_value(self):
         with self.assertRaises(ValidationException):
@@ -115,14 +104,14 @@ class TestSettings(SearxTestCase):
     def test_plugins_setting_all_default_enabled(self):
         plugin1 = PluginStub('plugin1', True)
         plugin2 = PluginStub('plugin2', True)
-        setting = PluginsSetting(['3'], choices=[plugin1, plugin2])
+        setting = PluginsSetting(['3'], plugins=[plugin1, plugin2])
         self.assertEqual(setting.get_enabled(), set(['plugin1', 'plugin2']))
 
     def test_plugins_setting_few_default_enabled(self):
         plugin1 = PluginStub('plugin1', True)
         plugin2 = PluginStub('plugin2', False)
         plugin3 = PluginStub('plugin3', True)
-        setting = PluginsSetting('name', choices=[plugin1, plugin2, plugin3])
+        setting = PluginsSetting('name', plugins=[plugin1, plugin2, plugin3])
         self.assertEqual(setting.get_enabled(), set(['plugin1', 'plugin3']))