Add searx_extra package

Split the utils directory into:
* searx_extra contains update scripts, standalone_searx.py
* utils contains the files to build and setup searx.

.github/workflows/data-update.yml

@@ -11,12 +11,12 @@ jobs:
     strategy:
       matrix:
         fetch:
-          - ahmia_blacklist
-          - currencies
-          - external_bangs
-          - firefox_version
-          - languages
-          - wikidata_units
+          - update_ahmia_blacklist.py
+          - update_currencies.py
+          - update_external_bangs.py
+          - update_firefox_version.py
+          - update_languages.py
+          - update_wikidata_units.py
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -45,10 +45,10 @@ jobs:
 
       - name: Fetch data
         env:
-          FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py
+          FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
         run: |
           source local/py3/bin/activate
-          python $FETCH_SCRIPT
+          $FETCH_SCRIPT
 
       - name: Create Pull Request
         id: cpr

Makefile

@@ -195,8 +195,8 @@ PYLINT_FILES=\
 	searx/engines/google_videos.py \
 	searx/engines/google_images.py \
 	searx/engines/mediathekviewweb.py \
-	utils/fetch_external_bangs.py \
-	searx/engines/google_scholar.py
+	searx/engines/google_scholar.py \
+	searx_extra/update/update_external_bangs.py
 
 test.pylint: pyenvinstall
 	$(call cmd,pylint,$(PYLINT_FILES))

docs/index.rst

@@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.
    user/index
    admin/index
    dev/index
+   searx_extra/index
    utils/index
    blog/index
 

docs/searx_extra/index.rst

@@ -0,0 +1,14 @@
+.. _searx_extra:
+
+======================================================
+Tooling box ``searx_extra`` for developers and users
+======================================================
+
+In the folder :origin:`searx_extra/` we maintain some tools useful for
+developers and users.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents
+
+   standalone_searx.py

docs/searx_extra/standalone_searx.py.rst

@@ -0,0 +1,9 @@
+
+.. _standalone_searx.py:
+
+===================================
+``searx_extra/standalone_searx.py``
+===================================
+
+.. automodule:: searx_extra.standalone_searx
+  :members:

docs/utils/index.rst

@@ -1,12 +1,11 @@
 .. _searx_utils:
 .. _toolboxing:
 
-=======================
-Tooling box ``utils/*``
-=======================
+========================================
+Tooling box ``utils`` for administrators
+========================================
 
-In the folder :origin:`utils/` we maintain some tools useful for admins and
-developers.
+In the folder :origin:`utils/` we maintain some tools useful for administrators.
 
 .. toctree::
    :maxdepth: 2
@@ -16,7 +15,6 @@ developers.
    filtron.sh
    morty.sh
    lxc.sh
-   standalone_searx.py
 
 .. _toolboxing common:
 

docs/utils/standalone_searx.py.rst

@@ -1,11 +0,0 @@
-
-.. _standalone_searx.py:
-
-=============================
-``utils/standalone_searx.py``
-=============================
-
-.. automodule:: standalone_searx
-  :members:
-
-

utils/standalone_searx.py → searx_extra/standalone_searx.py

@@ -15,7 +15,7 @@ Example to use this script:
 
 .. code::  bash
 
-    $ python3 utils/standalone_searx.py rain
+    $ python3 searx_extra/standalone_searx.py rain
 
 Example to run it from python:
 

utils/fetch_ahmia_blacklist.py → searx_extra/update/update_ahmia_blacklist.py

@@ -4,11 +4,8 @@
 # More info in https://ahmia.fi/blacklist/
 
 # set path
-from sys import path
-from os.path import realpath, dirname, join
-path.append(realpath(dirname(realpath(__file__)) + '/../'))
+from os.path import join
 
-#
 import requests
 from searx import searx_dir
 

utils/fetch_currencies.py → searx_extra/update/update_currencies.py

@@ -7,7 +7,6 @@ import json
 # set path
 from sys import path
 from os.path import realpath, dirname, join
-path.append(realpath(dirname(realpath(__file__)) + '/../'))
 
 from searx import searx_dir, settings
 from searx.engines.wikidata import send_wikidata_query

utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.py

@@ -3,13 +3,9 @@
 import sys
 import json
 from urllib.parse import quote, urlparse
-from os.path import realpath, dirname
-import cld3
+import detect_language
 from lxml.html import fromstring
 
-# set path
-sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
-
 from searx.engines.wikidata import send_wikidata_query
 from searx.utils import extract_text
 import searx

utils/fetch_external_bangs.py → searx_extra/update/update_external_bangs.py

@@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION
 """
 # pylint: disable=C0116
 
-import sys
 import json
 import re
-from os.path import realpath, dirname, join
+from os.path import join
 
 import requests
 
-# set path
-sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
-
 from searx import searx_dir  # pylint: disable=E0401 C0413
 
 

utils/fetch_firefox_version.py → searx_extra/update/update_firefox_version.py

@@ -1,14 +1,9 @@
 #!/usr/bin/env python
 
-# set path
-from sys import path
-from os.path import realpath, dirname, join
-path.append(realpath(dirname(realpath(__file__)) + '/../'))
-
-#
 import json
 import requests
 import re
+from os.path import dirname, join
 from urllib.parse import urlparse, urljoin
 from distutils.version import LooseVersion, StrictVersion
 from lxml import html

utils/fetch_languages.py → searx_extra/update/update_languages.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python
 
 # This script generates languages.py from intersecting each engine's supported languages.
 #
@@ -7,11 +7,9 @@
 import json
 from pathlib import Path
 from pprint import pformat
-from sys import path
 from babel import Locale, UnknownLocaleError
 from babel.languages import get_global
 
-path.append('../searx')  # noqa
 from searx import settings, searx_dir
 from searx.engines import initialize_engines, engines
 

utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.py

@@ -4,9 +4,7 @@ import json
 import collections
 
 # set path
-from sys import path
-from os.path import realpath, dirname, join
-path.append(realpath(dirname(realpath(__file__)) + '/../'))
+from os.path import join
 
 from searx import searx_dir
 from searx.engines.wikidata import send_wikidata_query

setup.py

@@ -41,7 +41,7 @@ setup(
     author='Adam Tauber',
     author_email='asciimoo@gmail.com',
     license='GNU Affero General Public License',
-    packages=find_packages(exclude=["tests*"]),
+    packages=find_packages(exclude=["tests*", "searx_extra"]),
     zip_safe=False,
     install_requires=requirements,
     extras_require={

tests/unit/test_standalone_searx.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 """Test utils/standalone_searx.py"""
 import datetime
-import importlib.util
 import io
 import sys
 
@@ -10,16 +9,7 @@ from nose2.tools import params
 
 from searx.search import SearchQuery, EngineRef, initialize
 from searx.testing import SearxTestCase
-
-
-def get_standalone_searx_module():
-    """Get standalone_searx module."""
-    module_name = 'utils.standalone_searx'
-    filename = 'utils/standalone_searx.py'
-    spec = importlib.util.spec_from_file_location(module_name, filename)
-    sas = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(sas)
-    return sas
+from searx_extra import standalone_searx as sas
 
 
 class StandaloneSearx(SearxTestCase):
@@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):
 
     def test_parse_argument_no_args(self):
         """Test parse argument without args."""
-        sas = get_standalone_searx_module()
         with patch.object(sys, 'argv', ['standalone_searx']), \
                 self.assertRaises(SystemExit):
             sys.stderr = io.StringIO()
@@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):
 
     def test_parse_argument_basic_args(self):
         """Test parse argument with basic args."""
-        sas = get_standalone_searx_module()
         query = 'red box'
         exp_dict = {
             'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
@@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):
 
     def test_to_dict(self):
         """test to_dict."""
-        sas = get_standalone_searx_module()
         self.assertEqual(
             sas.to_dict(
                 sas.get_search_query(sas.parse_argument(['red box']))),
@@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):
 
     def test_to_dict_with_mock(self):
         """test to dict."""
-        sas = get_standalone_searx_module()
         with patch.object(sas.searx.search, 'Search') as mock_s:
             m_search = mock_s().search()
             m_sq = Mock()
@@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):
 
     def test_get_search_query(self):
         """test get_search_query."""
-        sas = get_standalone_searx_module()
         args = sas.parse_argument(['rain', ])
         search_q = sas.get_search_query(args)
         self.assertTrue(search_q)
@@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):
 
     def test_no_parsed_url(self):
         """test no_parsed_url func"""
-        sas = get_standalone_searx_module()
         self.assertEqual(
             sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
             [{}]
@@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):
     )
     def test_json_serial(self, arg, exp_res):
         """test json_serial func"""
-        sas = get_standalone_searx_module()
         self.assertEqual(sas.json_serial(arg), exp_res)
 
     def test_json_serial_error(self):
         """test error on json_serial."""
-        sas = get_standalone_searx_module()
         with self.assertRaises(TypeError):
             sas.json_serial('a')