scheduler.py 2.5 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758
  1. # SPDX-License-Identifier: AGPL-3.0-or-later
  2. # pylint: disable=missing-module-docstring
  3. """Lame scheduler which use Valkey as a source of truth:
  4. * the Valkey key SearXNG_checker_next_call_ts contains the next time the embedded checker should run.
  5. * to avoid lock, a unique Valkey script reads and updates the Valkey key SearXNG_checker_next_call_ts.
  6. * this Valkey script returns a list of two elements:
  7. * the first one is a boolean. If True, the embedded checker must run now in this worker.
  8. * the second element is the delay in second to wait before the next call to the Valkey script.
  9. This scheduler is not generic on purpose: if more feature are required, a dedicate scheduler must be used
  10. (= a better scheduler should not use the web workers)
  11. """
  12. import logging
  13. import time
  14. from pathlib import Path
  15. from typing import Callable
  16. from searx.valkeydb import client as get_valkey_client
  17. from searx.valkeylib import lua_script_storage
  18. logger = logging.getLogger('searx.search.checker')
  19. SCHEDULER_LUA = Path(__file__).parent / "scheduler.lua"
  20. def scheduler_function(start_after_from: int, start_after_to: int, every_from: int, every_to: int, callback: Callable):
  21. """Run the checker periodically. The function never returns.
  22. Parameters:
  23. * start_after_from and start_after_to: when to call "callback" for the first on the Valkey instance
  24. * every_from and every_to: after the first call, how often to call "callback"
  25. There is no issue:
  26. * to call this function is multiple workers
  27. * to kill workers at any time as long there is one at least one worker
  28. """
  29. scheduler_now_script = SCHEDULER_LUA.open().read()
  30. while True:
  31. # ask the Valkey script what to do
  32. # the script says
  33. # * if the checker must run now.
  34. # * how to long to way before calling the script again (it can be call earlier, but not later).
  35. script = lua_script_storage(get_valkey_client(), scheduler_now_script)
  36. call_now, wait_time = script(args=[start_after_from, start_after_to, every_from, every_to])
  37. # does the worker run the checker now?
  38. if call_now:
  39. # run the checker
  40. try:
  41. callback()
  42. except Exception: # pylint: disable=broad-except
  43. logger.exception("Error calling the embedded checker")
  44. # only worker display the wait_time
  45. logger.info("Next call to the checker in %s seconds", wait_time)
  46. # wait until the next call
  47. time.sleep(wait_time)