Browse Source

Merge branch 'master' of https://github.com/asciimoo/searx into filtron

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
Markus Heiser 5 years ago
parent
commit
4d67164918
7 changed files with 84 additions and 17 deletions
  1. 1 1
      docs/conf.py
  2. 1 1
      docs/dev/reST.rst
  3. 4 2
      docs/index.rst
  4. 54 0
      searx/engines/etools.py
  5. 7 2
      searx/preferences.py
  6. 6 0
      searx/settings.yml
  7. 11 11
      searx/webapp.py

+ 1 - 1
docs/conf.py

@@ -12,7 +12,7 @@ DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/")
 # Project --------------------------------------------------------------
 # Project --------------------------------------------------------------
 
 
 project = u'searx'
 project = u'searx'
-copyright = u'2015-2019, Adam Tauber, Noémi Ványi'
+copyright = u'2015-2020, Adam Tauber, Noémi Ványi'
 author = u'Adam Tauber'
 author = u'Adam Tauber'
 release, version = VERSION_STRING, VERSION_STRING
 release, version = VERSION_STRING, VERSION_STRING
 highlight_language = 'none'
 highlight_language = 'none'

+ 1 - 1
docs/dev/reST.rst

@@ -285,7 +285,7 @@ content becomes smart.
    :rst:role:`pep`            :pep:`8`                           ``:pep:`8```
    :rst:role:`pep`            :pep:`8`                           ``:pep:`8```
    sphinx.ext.extlinks_
    sphinx.ext.extlinks_
    --------------------------------------------------------------------------------------------------
    --------------------------------------------------------------------------------------------------
-   project's wiki article     :wiki:`Searx-instances`            ``:wiki:`Searx-instances```
+   project's wiki article     :wiki:`Offline-engines`            ``:wiki:`Offline-engines```
    to docs public URL         :docs:`dev/reST.html`              ``:docs:`dev/reST.html```
    to docs public URL         :docs:`dev/reST.html`              ``:docs:`dev/reST.html```
    files & folders origin     :origin:`docs/dev/reST.rst`        ``:origin:`docs/dev/reST.rst```
    files & folders origin     :origin:`docs/dev/reST.rst`        ``:origin:`docs/dev/reST.rst```
    pull request               :pull:`1756`                       ``:pull:`1756```
    pull request               :pull:`1756`                       ``:pull:`1756```

+ 4 - 2
docs/index.rst

@@ -8,8 +8,8 @@ Searx is a free internet metasearch engine which aggregates results from more
 than 70 search services.  Users are neither tracked nor profiled.  Additionally,
 than 70 search services.  Users are neither tracked nor profiled.  Additionally,
 searx can be used over Tor for online anonymity.
 searx can be used over Tor for online anonymity.
 
 
-Get started with searx by using one of the :wiki:`Searx-instances`.  If you
+Get started with searx by using one of the Searx-instances_.  If you don't trust
-don't trust anyone, you can set up your own, see :ref:`installation`.
+anyone, you can set up your own, see :ref:`installation`.
 
 
 .. sidebar::  Features
 .. sidebar::  Features
 
 
@@ -32,3 +32,5 @@ don't trust anyone, you can set up your own, see :ref:`installation`.
    dev/index
    dev/index
    utils/index
    utils/index
    blog/index
    blog/index
+
+.. _Searx-instances: https://searx.space

+ 54 - 0
searx/engines/etools.py

@@ -0,0 +1,54 @@
+"""
+ eTools (Web)
+
+ @website      https://www.etools.ch
+ @provide-api  no
+ @using-api    no
+ @results      HTML
+ @stable       no (HTML can change)
+ @parse        url, title, content
+"""
+
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import quote
+from searx.utils import eval_xpath
+
+categories = ['general']
+paging = False
+language_support = False
+safesearch = True
+
+base_url = 'https://www.etools.ch'
+search_path = '/searchAdvancedSubmit.do'\
+    '?query={search_term}'\
+    '&pageResults=20'\
+    '&safeSearch={safesearch}'
+
+
+def request(query, params):
+    if params['safesearch']:
+        safesearch = 'true'
+    else:
+        safesearch = 'false'
+
+    params['url'] = base_url + search_path.format(search_term=quote(query), safesearch=safesearch)
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    for result in eval_xpath(dom, '//table[@class="result"]//td[@class="record"]'):
+        url = eval_xpath(result, './a/@href')[0]
+        title = extract_text(eval_xpath(result, './a//text()'))
+        content = extract_text(eval_xpath(result, './/div[@class="text"]//text()'))
+
+        results.append({'url': url,
+                        'title': title,
+                        'content': content})
+
+    return results

+ 7 - 2
searx/preferences.py

@@ -4,6 +4,7 @@ from sys import version
 
 
 from searx import settings, autocomplete
 from searx import settings, autocomplete
 from searx.languages import language_codes as languages
 from searx.languages import language_codes as languages
+from searx.utils import match_language
 from searx.url_utils import parse_qs, urlencode
 from searx.url_utils import parse_qs, urlencode
 
 
 if version[0] == '3':
 if version[0] == '3':
@@ -11,7 +12,7 @@ if version[0] == '3':
 
 
 
 
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
-LANGUAGE_CODES = [l[0].split('-')[0] for l in languages]
+LANGUAGE_CODES = [l[0] for l in languages]
 LANGUAGE_CODES.append('all')
 LANGUAGE_CODES.append('all')
 DISABLED = 0
 DISABLED = 0
 ENABLED = 1
 ENABLED = 1
@@ -132,6 +133,10 @@ class SetSetting(Setting):
 class SearchLanguageSetting(EnumStringSetting):
 class SearchLanguageSetting(EnumStringSetting):
     """Available choices may change, so user's value may not be in choices anymore"""
     """Available choices may change, so user's value may not be in choices anymore"""
 
 
+    def _validate_selection(self, selection):
+        if not match_language(selection, self.choices, fallback=None) and selection != "":
+            raise ValidationException('Invalid language code: "{0}"'.format(selection))
+
     def parse(self, data):
     def parse(self, data):
         if data not in self.choices and data != self.value:
         if data not in self.choices and data != self.value:
             # hack to give some backwards compatibility with old language cookies
             # hack to give some backwards compatibility with old language cookies
@@ -268,7 +273,7 @@ class Preferences(object):
         super(Preferences, self).__init__()
         super(Preferences, self).__init__()
 
 
         self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']),
         self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']),
-                                   'language': SearchLanguageSetting(settings['ui']['default_locale'],
+                                   'language': SearchLanguageSetting(settings['search']['default_lang'],
                                                                      choices=list(LANGUAGE_CODES) + ['']),
                                                                      choices=list(LANGUAGE_CODES) + ['']),
                                    'locale': EnumStringSetting(settings['ui']['default_locale'],
                                    'locale': EnumStringSetting(settings['ui']['default_locale'],
                                                                choices=list(settings['locales'].keys()) + ['']),
                                                                choices=list(settings['locales'].keys()) + ['']),

+ 6 - 0
searx/settings.yml

@@ -5,6 +5,7 @@ general:
 search:
 search:
     safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
     safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
     autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
     autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
+    default_lang : "" # Default search language - leave blank to detect from browser information or use codes from 'languages.py'
     ban_time_on_fail : 5 # ban time in seconds after engine errors
     ban_time_on_fail : 5 # ban time in seconds after engine errors
     max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
     max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
 
 
@@ -202,6 +203,11 @@ engines:
     timeout: 3.0
     timeout: 3.0
     disabled : True
     disabled : True
 
 
+  - name : etools
+    engine : etools
+    shortcut : eto
+    disabled : True
+
   - name : etymonline
   - name : etymonline
     engine : xpath
     engine : xpath
     paging : True
     paging : True

+ 11 - 11
searx/webapp.py

@@ -157,14 +157,16 @@ _category_names = (gettext('files'),
 outgoing_proxies = settings['outgoing'].get('proxies') or None
 outgoing_proxies = settings['outgoing'].get('proxies') or None
 
 
 
 
+def _get_browser_language(request, lang_list):
+    for lang in request.headers.get("Accept-Language", "en").split(","):
+        locale = match_language(lang, lang_list, fallback=None)
+        if locale is not None:
+            return locale
+
+
 @babel.localeselector
 @babel.localeselector
 def get_locale():
 def get_locale():
-    locale = "en-US"
+    locale = _get_browser_language(request, settings['locales'].keys())
-
-    for lang in request.headers.get("Accept-Language", locale).split(","):
-        locale = match_language(lang, settings['locales'].keys(), fallback=None)
-        if locale is not None:
-            break
 
 
     logger.debug("default locale from browser info is `%s`", locale)
     logger.debug("default locale from browser info is `%s`", locale)
 
 
@@ -372,8 +374,7 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['language_codes'] = languages
     kwargs['language_codes'] = languages
     if 'current_language' not in kwargs:
     if 'current_language' not in kwargs:
         kwargs['current_language'] = match_language(request.preferences.get_value('language'),
         kwargs['current_language'] = match_language(request.preferences.get_value('language'),
-                                                    LANGUAGE_CODES,
+                                                    LANGUAGE_CODES)
-                                                    fallback=locale)
 
 
     # override url_for function in templates
     # override url_for function in templates
     kwargs['url_for'] = url_for_theme
     kwargs['url_for'] = url_for_theme
@@ -444,11 +445,10 @@ def pre_request():
             request.errors.append(gettext('Invalid settings'))
             request.errors.append(gettext('Invalid settings'))
 
 
     # init search language and locale
     # init search language and locale
-    locale = get_locale()
     if not preferences.get_value("language"):
     if not preferences.get_value("language"):
-        preferences.parse_dict({"language": locale})
+        preferences.parse_dict({"language": _get_browser_language(request, LANGUAGE_CODES)})
     if not preferences.get_value("locale"):
     if not preferences.get_value("locale"):
-        preferences.parse_dict({"locale": locale})
+        preferences.parse_dict({"locale": get_locale()})
 
 
     # request.user_plugins
     # request.user_plugins
     request.user_plugins = []
     request.user_plugins = []