[fix] issues when launching a local development server

A local development server can be launched by one of these command lines::

    $ flask --app searx.webapp run
    $ python -m searx.webapp

The different ways of starting the server should lead to the same result, which
is generally the case.  However, if the modules are reloaded after code
changes (reload option), it must be avoided that the application is initialized
twice at startup.  We have already discussed this in 2022 [1][2].

Further information on this topic can be found in [3][4][5].

To test a bash in the ./local environment was started and the follwing commands
had been executed::

    $ ./manage pyenv.cmd bash --norc --noprofile
    (py3) SEARXNG_DEBUG=1 flask --app searx.webapp run --reload
    (py3) SEARXNG_DEBUG=1 python -m searx.webapp

Since the generic parts of the docs also initialize the app to generate doc from
it, the build of the docs was also tested::

    $ make docs.clean docs.live

[1] https://github.com/searxng/searxng/pull/1656#issuecomment-1214198941
[2] https://github.com/searxng/searxng/pull/1616#issuecomment-1206137468
[3] https://flask.palletsprojects.com/en/stable/api/#flask.Flask.run
[4] https://github.com/pallets/flask/issues/5307#issuecomment-1774646119
[5] https://stackoverflow.com/a/25504196

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

docs/admin/settings/settings_general.rst

@@ -16,8 +16,14 @@
      open_metrics: ''
 
 ``debug`` : ``$SEARXNG_DEBUG``
-  Allow a more detailed log if you run SearXNG directly. Display *detailed* error
-  messages in the browser too, so this must be deactivated in production.
+  In debug mode, the server provides an interactive debugger, will reload when
+  code is changed and activates a verbose logging.
+
+  .. attention::
+
+     The debug setting is intended for local development server.  Don't
+     activate debug (don't use a development server) when deploying to
+     production.
 
 ``donation_url`` :
   Set value to ``true`` to use your own donation page written in the

searx/__init__.py

@@ -22,18 +22,18 @@ searx_dir = abspath(dirname(__file__))
 searx_parent_dir = abspath(dirname(dirname(__file__)))
 
 settings = {}
-searx_debug = False
+sxng_debug = False
 logger = logging.getLogger('searx')
 
 _unset = object()
 
 
 def init_settings():
-    """Initialize global ``settings`` and ``searx_debug`` variables and
+    """Initialize global ``settings`` and ``sxng_debug`` variables and
     ``logger`` from ``SEARXNG_SETTINGS_PATH``.
     """
 
-    global settings, searx_debug  # pylint: disable=global-variable-not-assigned
+    global settings, sxng_debug  # pylint: disable=global-variable-not-assigned
 
     cfg, msg = searx.settings_loader.load_settings(load_user_settings=True)
     cfg = cfg or {}
@@ -42,8 +42,8 @@ def init_settings():
     settings.clear()
     settings.update(cfg)
 
-    searx_debug = settings['general']['debug']
-    if searx_debug:
+    sxng_debug = get_setting("general.debug")
+    if sxng_debug:
         _logging_config_debug()
     else:
         logging.basicConfig(level=LOG_LEVEL_PROD, format=LOG_FORMAT_PROD)

searx/network/network.py

@@ -12,7 +12,7 @@ from typing import Dict
 
 import httpx
 
-from searx import logger, searx_debug
+from searx import logger, sxng_debug
 from searx.extended_types import SXNG_Response
 from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
 from .raise_for_httperror import raise_for_httperror
@@ -186,7 +186,7 @@ class Network:
         local_address = next(self._local_addresses_cycle)
         proxies = next(self._proxies_cycle)  # is a tuple so it can be part of the key
         key = (verify, max_redirects, local_address, proxies)
-        hook_log_response = self.log_response if searx_debug else None
+        hook_log_response = self.log_response if sxng_debug else None
         if key not in self._clients or self._clients[key].is_closed:
             client = new_client(
                 self.enable_http,

searx/search/checker/background.py

@@ -10,7 +10,7 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
 
 import redis.exceptions
 
-from searx import logger, settings, searx_debug
+from searx import logger, settings, sxng_debug
 from searx.redisdb import client as get_redis_client
 from searx.exceptions import SearxSettingsException
 from searx.search.processors import PROCESSORS
@@ -139,7 +139,7 @@ def initialize():
         signal.signal(signal.SIGUSR1, _signal_handler)
 
     # special case when debug is activate
-    if searx_debug and settings['checker']['off_when_debug']:
+    if sxng_debug and settings['checker']['off_when_debug']:
         logger.info('debug mode: checker is disabled')
         return
 

searx/webapp.py

@@ -6,6 +6,7 @@
 # pylint: disable=use-dict-literal
 from __future__ import annotations
 
+import inspect
 import hashlib
 import hmac
 import json
@@ -29,6 +30,8 @@ from pygments import highlight
 from pygments.lexers import get_lexer_by_name
 from pygments.formatters import HtmlFormatter  # pylint: disable=no-name-in-module
 
+from werkzeug.serving import is_running_from_reloader
+
 import flask
 
 from flask import (
@@ -48,12 +51,12 @@ from flask_babel import (
     format_decimal,
 )
 
+import searx
 from searx.extended_types import sxng_request
 from searx import (
     logger,
     get_setting,
     settings,
-    searx_debug,
 )
 
 from searx import infopage
@@ -81,7 +84,6 @@ from searx.webutils import (
     exception_classname_to_text,
     new_hmac,
     is_hmac_of,
-    is_flask_run_cmdline,
     group_engines_in_tab,
 )
 from searx.webadapter import (
@@ -128,11 +130,6 @@ logger = logger.getChild('webapp')
 
 warnings.simplefilter("always")
 
-# check secret_key
-if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey':
-    logger.error('server.secret_key is not changed. Please use something else instead of ultrasecretkey.')
-    sys.exit(1)
-
 # about static
 logger.debug('static directory is %s', settings['ui']['static_path'])
 static_files = get_static_files(settings['ui']['static_path'])
@@ -1329,45 +1326,108 @@ def page_not_found(_e):
     return render('404.html'), 404
 
 
-# see https://flask.palletsprojects.com/en/1.1.x/cli/
-# True if "FLASK_APP=searx/webapp.py FLASK_ENV=development flask run"
-flask_run_development = (
-    os.environ.get("FLASK_APP") is not None and os.environ.get("FLASK_ENV") == 'development' and is_flask_run_cmdline()
-)
+def run():
+    """Runs the application on a local development server.
+
+    This run method is only called when SearXNG is started via ``__main__``::
+
+        python -m searx.webapp
+
+    Do not use :ref:`run() <flask.Flask.run>` in a production setting.  It is
+    not intended to meet security and performance requirements for a production
+    server.
+
+    It is not recommended to use this function for development with automatic
+    reloading as this is badly supported.  Instead you should be using the flask
+    command line script’s run support::
+
+        flask --app searx.webapp run --debug --reload --host 127.0.0.1 --port 8888
+
+    .. _Flask.run: https://flask.palletsprojects.com/en/stable/api/#flask.Flask.run
+    """
+
+    host: str = get_setting("server.bind_address")  # type: ignore
+    port: int = get_setting("server.port")  # type: ignore
+
+    if searx.sxng_debug:
+        logger.debug("run local development server (DEBUG) on %s:%s", host, port)
+        app.run(
+            debug=True,
+            port=port,
+            host=host,
+            threaded=True,
+            extra_files=[DEFAULT_SETTINGS_FILE],
+        )
+    else:
+        logger.debug("run local development server on %s:%s", host, port)
+        app.run(port=port, host=host, threaded=True)
+
+
+def is_werkzeug_reload_active() -> bool:
+    """Returns ``True`` if server is is launched by :ref:`werkzeug.serving` and
+    the ``use_reload`` argument was set to ``True``.  If this is the case, it
+    should be avoided that the server is initialized twice (:py:obj:`init`,
+    :py:obj:`run`).
+
+    .. _werkzeug.serving:
+       https://werkzeug.palletsprojects.com/en/stable/serving/#werkzeug.serving.run_simple
+    """
+
+    # https://github.com/searxng/searxng/pull/1656#issuecomment-1214198941
+    # https://github.com/searxng/searxng/pull/1616#issuecomment-1206137468
 
-# True if reload feature is activated of werkzeug, False otherwise (including uwsgi, etc..)
-#  __name__ != "__main__" if searx.webapp is imported (make test, make docs, uwsgi...)
-# see run() at the end of this file : searx_debug activates the reload feature.
-werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__main__")
+    frames = inspect.stack()
+
+    if len(frames) > 1 and frames[-2].filename.endswith('flask/cli.py'):
+        # server was launched by "flask run", is argument "--reload" set?
+        if "--reload" in sys.argv or "--debug" in sys.argv:
+            return True
+
+    elif frames[0].filename.endswith('searx/webapp.py'):
+        # server was launched by "python -m searx.webapp" / see run()
+        if searx.sxng_debug:
+            return True
+
+    return False
+
+
+def init():
+
+    if searx.sxng_debug or app.debug:
+        app.debug = True
+        searx.sxng_debug = True
+
+    # check secret_key in production
+
+    if not app.debug and get_setting("server.secret_key") == 'ultrasecretkey':
+        logger.error("server.secret_key is not changed. Please use something else instead of ultrasecretkey.")
+        sys.exit(1)
+
+    # When automatic reloading is activated stop Flask from initialising twice.
+    # - https://github.com/pallets/flask/issues/5307#issuecomment-1774646119
+    # - https://stackoverflow.com/a/25504196
+
+    reloader_active = is_werkzeug_reload_active()
+    werkzeug_run_main = is_running_from_reloader()
+
+    if reloader_active and not werkzeug_run_main:
+        logger.info("in reloading mode and not in main loop, cancel the initialization")
+        return
 
-# initialize the engines except on the first run of the werkzeug server.
-if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
     locales_initialize()
     redis_initialize()
     searx.plugins.initialize(app)
-    searx.search.initialize(
-        enable_checker=True,
-        check_network=True,
-        enable_metrics=get_setting("general.enable_metrics"),
-    )
-    limiter.initialize(app, settings)
-    favicons.init()
 
+    metrics: bool = get_setting("general.enable_metrics")  # type: ignore
+    searx.search.initialize(enable_checker=True, check_network=True, enable_metrics=metrics)
 
-def run():
-    logger.debug('starting webserver on %s:%s', settings['server']['bind_address'], settings['server']['port'])
-    app.run(
-        debug=searx_debug,
-        use_debugger=searx_debug,
-        port=settings['server']['port'],
-        host=settings['server']['bind_address'],
-        threaded=True,
-        extra_files=[DEFAULT_SETTINGS_FILE],
-    )
+    limiter.initialize(app, settings)
+    favicons.init()
 
 
 application = app
 patch_application(app)
+init()
 
 if __name__ == "__main__":
     run()

searx/webutils.py

@@ -9,7 +9,6 @@ import csv
 import hashlib
 import hmac
 import re
-import inspect
 import itertools
 import json
 from datetime import datetime, timedelta
@@ -316,21 +315,6 @@ def searxng_l10n_timespan(dt: datetime) -> str:  # pylint: disable=invalid-name
     return format_date(dt)
 
 
-def is_flask_run_cmdline():
-    """Check if the application was started using "flask run" command line
-
-    Inspect the callstack.
-    See https://github.com/pallets/flask/blob/master/src/flask/__main__.py
-
-    Returns:
-        bool: True if the application was started using "flask run".
-    """
-    frames = inspect.stack()
-    if len(frames) < 2:
-        return False
-    return frames[-2].filename.endswith('flask/cli.py')
-
-
 NO_SUBGROUPING = 'without further subgrouping'
 
 

tests/__init__.py

@@ -5,13 +5,8 @@ import pathlib
 import os
 import aiounittest
 
-# Before import from the searx package, we need to set up the (debug)
-# environment.  The import of the searx package initialize the searx.settings
-# and this in turn takes the defaults from the environment!
 
 os.environ.pop('SEARXNG_SETTINGS_PATH', None)
-os.environ['SEARXNG_DEBUG'] = '1'
-os.environ['SEARXNG_DEBUG_LOG_LEVEL'] = 'WARNING'
 os.environ['SEARXNG_DISABLE_ETC_SETTINGS'] = '1'
 
 

tests/robot/__main__.py

@@ -27,15 +27,6 @@ class SearxRobotLayer:
         webapp = str(tests_path.parent / 'searx' / 'webapp.py')
         exe = 'python'
 
-        # The Flask app is started by Flask.run(...), don't enable Flask's debug
-        # mode, the debugger from Flask will cause wired process model, where
-        # the server never dies.  Further read:
-        #
-        # - debug mode: https://flask.palletsprojects.com/quickstart/#debug-mode
-        # - Flask.run(..): https://flask.palletsprojects.com/api/#flask.Flask.run
-
-        os.environ['SEARXNG_DEBUG'] = '0'
-
         # set robot settings path
         os.environ['SEARXNG_SETTINGS_PATH'] = str(tests_path / 'robot' / 'settings_robot.yml')