Merge branch 'master' into http1.1

.travis.yml

@@ -4,6 +4,8 @@ cache:
   - npm
   - directories:
     - $HOME/.cache/pip
+addons:
+  firefox: "latest"
 language: python
 python:
   - "2.7"
@@ -12,6 +14,9 @@ before_install:
   - "sh -e /etc/init.d/xvfb start"
   - npm install less grunt-cli
   - ( cd searx/static/themes/oscar;npm install; cd - )
+  - mkdir -p ~/drivers; export PATH=~/drivers:$PATH;
+  - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-linux64.tar.gz";
+  - FILE=`mktemp`; wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C ~/drivers -f $FILE geckodriver; rm $FILE; chmod 777 ~/drivers/geckodriver;
 install:
   - ./manage.sh update_dev_packages
   - pip install coveralls

AUTHORS.rst

@@ -58,3 +58,5 @@ generally made searx better:
 - marc @a01200356
 - Harry Wood @harry-wood
 - Thomas Renard @threnard
+- Pydo `<https://github.com/pydo>`_
+- Athemis `<https://github.com/Athemis>`_

manage.sh

@@ -53,8 +53,8 @@ build_style() {
 
 styles() {
     echo '[!] Building styles'
-	build_style themes/default/less/style.less themes/default/css/style.css
-	build_style themes/default/less/style-rtl.less themes/default/css/style-rtl.css
+	build_style themes/legacy/less/style.less themes/legacy/css/style.css
+	build_style themes/legacy/less/style-rtl.less themes/legacy/css/style-rtl.css
 	build_style themes/courgette/less/style.less themes/courgette/css/style.css
 	build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css
 	build_style less/bootstrap/bootstrap.less css/bootstrap.min.css

requirements-dev.txt

@@ -3,8 +3,8 @@ mock==2.0.0
 nose2[coverage-plugin]
 pep8==1.7.0
 plone.testing==5.0.0
-robotframework-selenium2library==1.7.4
+robotframework-selenium2library==1.8.0
 robotsuite==1.7.0
-transifex-client==0.11
+transifex-client==0.12.2
 unittest2==1.1.0
-zope.testrunner==4.4.10
+zope.testrunner==4.5.1

requirements.txt

@@ -1,4 +1,4 @@
-certifi==2016.2.28
+certifi==2016.9.26
 flask==0.11.1
 flask-babel==0.11.1
 lxml==3.6.0

searx/engines/__init__.py

@@ -57,11 +57,17 @@ def load_module(filename):
 
 
 def load_engine(engine_data):
-    engine_name = engine_data['engine']
+
+    if '_' in engine_data['name']:
+        logger.error('Engine name conains underscore: "{}"'.format(engine_data['name']))
+        sys.exit(1)
+
+    engine_module = engine_data['engine']
+
     try:
-        engine = load_module(engine_name + '.py')
+        engine = load_module(engine_module + '.py')
     except:
-        logger.exception('Cannot load engine "{}"'.format(engine_name))
+        logger.exception('Cannot load engine "{}"'.format(engine_module))
         return None
 
     for param_name in engine_data:

searx/engines/digbt.py

@@ -40,7 +40,7 @@ def response(resp):
     results = list()
     for result in search_res:
         url = urljoin(URL, result.xpath('.//a[@title]/@href')[0])
-        title = result.xpath('.//a[@title]/text()')[0]
+        title = extract_text(result.xpath('.//a[@title]'))
         content = extract_text(result.xpath('.//div[@class="files"]'))
         files_data = extract_text(result.xpath('.//div[@class="tail"]')).split()
         filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER])

searx/engines/kickass.py

@@ -16,13 +16,14 @@ from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size, convert_str_to_int
 
 # engine dependent config
 categories = ['videos', 'music', 'files']
 paging = True
 
 # search-url
-url = 'https://kickass.to/'
+url = 'https://kickass.cd/'
 search_url = url + 'search/{search_term}/{pageno}/'
 
 # specific xpath variables
@@ -57,41 +58,16 @@ def response(resp):
         href = urljoin(url, link.attrib['href'])
         title = extract_text(link)
         content = escape(extract_text(result.xpath(content_xpath)))
-        seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
-        leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
-        filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
-        filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0]
-        files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0]
-
-        # convert seed to int if possible
-        if seed.isdigit():
-            seed = int(seed)
-        else:
-            seed = 0
+        seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
+        leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
+        filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))
+        files = extract_text(result.xpath('.//td[contains(@class, "center")][2]'))
 
-        # convert leech to int if possible
-        if leech.isdigit():
-            leech = int(leech)
-        else:
-            leech = 0
-
-        # convert filesize to byte if possible
-        try:
-            filesize = float(filesize)
-
-            # convert filesize to byte
-            if filesize_multiplier == 'TB':
-                filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'GB':
-                filesize = int(filesize * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'MB':
-                filesize = int(filesize * 1024 * 1024)
-            elif filesize_multiplier == 'KB':
-                filesize = int(filesize * 1024)
-        except:
-            filesize = None
-
-        # convert files to int if possible
+        seed = convert_str_to_int(seed)
+        leech = convert_str_to_int(leech)
+
+        filesize, filesize_multiplier = filesize_info.split()
+        filesize = get_torrent_size(filesize, filesize_multiplier)
         if files.isdigit():
             files = int(files)
         else:

searx/engines/pdbe.py

@@ -0,0 +1,109 @@
+"""
+ PDBe (Protein Data Bank in Europe)
+
+ @website       https://www.ebi.ac.uk/pdbe
+ @provide-api   yes (https://www.ebi.ac.uk/pdbe/api/doc/search.html),
+                unlimited
+ @using-api     yes
+ @results       python dictionary (from json)
+ @stable        yes
+ @parse         url, title, content, img_src
+"""
+
+from json import loads
+from flask_babel import gettext
+
+categories = ['science']
+
+hide_obsolete = False
+
+# status codes of unpublished entries
+pdb_unpublished_codes = ['HPUB', 'HOLD', 'PROC', 'WAIT', 'AUTH', 'AUCO', 'REPL', 'POLC', 'REFI', 'TRSF', 'WDRN']
+# url for api query
+pdbe_solr_url = 'https://www.ebi.ac.uk/pdbe/search/pdb/select?'
+# base url for results
+pdbe_entry_url = 'https://www.ebi.ac.uk/pdbe/entry/pdb/{pdb_id}'
+# link to preview image of structure
+pdbe_preview_url = 'https://www.ebi.ac.uk/pdbe/static/entry/{pdb_id}_deposited_chain_front_image-200x200.png'
+
+
+def request(query, params):
+
+    params['url'] = pdbe_solr_url
+    params['method'] = 'POST'
+    params['data'] = {
+        'q': query,
+        'wt': "json"  # request response in parsable format
+    }
+    return params
+
+
+def construct_body(result):
+    # set title
+    title = result['title']
+
+    # construct content body
+    content = """{title}<br />{authors} {journal} <strong>{volume}</strong>&nbsp;{page} ({year})"""
+
+    # replace placeholders with actual content
+    try:
+        if result['journal']:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal=result['journal'], volume=result['journal_volume'],
+                page=result['journal_page'], year=result['citation_year'])
+        else:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal='', volume='', page='', year=result['release_year'])
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        content = None
+        img_src = None
+
+    # construct url for preview image
+    try:
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        img_src = None
+
+    return [title, content, img_src]
+
+
+def response(resp):
+
+    results = []
+    json = loads(resp.text)['response']['docs']
+
+    # parse results
+    for result in json:
+        # catch obsolete entries and mark them accordingly
+        if result['status'] in pdb_unpublished_codes:
+            continue
+        if hide_obsolete:
+            continue
+        if result['status'] == 'OBS':
+            # expand title to add some sort of warning message
+            title = gettext('{title}&nbsp;(OBSOLETE)').format(title=result['title'])
+            superseded_url = pdbe_entry_url.format(pdb_id=result['superseded_by'])
+
+            # since we can't construct a proper body from the response, we'll make up our own
+            msg_superseded = gettext("This entry has been superseded by")
+            content = '<em>{msg_superseded} \<a href="{url}">{pdb_id}</a></em>'.format(
+                msg_superseded=msg_superseded,
+                url=superseded_url,
+                pdb_id=result['superseded_by'], )
+
+            # obsoleted entries don't have preview images
+            img_src = None
+        else:
+            title, content, img_src = construct_body(result)
+
+        results.append({
+            'url': pdbe_entry_url.format(pdb_id=result['pdb_id']),
+            'title': title,
+            'content': content,
+            'img_src': img_src
+        })
+
+    return results

searx/engines/seedpeer.py

@@ -0,0 +1,78 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     http://seedpeer.eu
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import quote
+from lxml import html
+from operator import itemgetter
+from searx.engines.xpath import extract_text
+
+
+url = 'http://www.seedpeer.eu/'
+search_url = url + 'search/{search_term}/7/{page_no}.html'
+# specific xpath variables
+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'] - 1)
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    torrent_links = dom.xpath(torrent_xpath)
+    if len(torrent_links) > 0:
+        seeds = dom.xpath(seeds_xpath)
+        peers = dom.xpath(peers_xpath)
+        titles = dom.xpath(title_xpath)
+        sizes = dom.xpath(size_xpath)
+        ages = dom.xpath(age_xpath)
+    else:  # under ~5 results uses a different xpath
+        torrent_links = dom.xpath(alternative_torrent_xpath)
+        seeds = dom.xpath(alternative_seeds_xpath)
+        peers = dom.xpath(alternative_peers_xpath)
+        titles = dom.xpath(alternative_title_xpath)
+        sizes = dom.xpath(alternative_size_xpath)
+        ages = dom.xpath(alternative_age_xpath)
+    # return empty array if nothing is found
+    if not torrent_links:
+        return []
+
+    # parse results
+    for index, result in enumerate(torrent_links):
+        link = result.attrib.get('href')
+        href = urljoin(url, link)
+        results.append({'url': href,
+                        'title': titles[index].text_content(),
+                        'content': '{}, {}'.format(sizes[index], ages[index]),
+                        'seed': seeds[index],
+                        'leech': peers[index],
+
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)

searx/settings.yml

@@ -18,6 +18,12 @@ ui:
     default_theme : oscar # ui theme
     default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
 
+# searx supports result proxification using an external service: https://github.com/asciimoo/morty
+# uncomment below section if you have running morty proxy
+#result_proxy:
+#    url : http://127.0.0.1:3000/
+#    key : your_morty_proxy_key
+
 outgoing: # communication with search engines
     request_timeout : 2.0 # seconds
     useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
@@ -301,6 +307,12 @@ engines:
     timeout : 6.0
     disabled : True
 
+  - name: kickass
+    engine : kickass
+    shortcut : kc
+    timeout : 4.0
+    disabled : True
+
   - name : microsoft academic
     engine : json_engine
     paging : True
@@ -339,6 +351,13 @@ engines:
     disabled : True
     shortcut : or
 
+  - name : pdbe
+    engine : pdbe
+    shortcut : pdb
+# Hide obsolete PDB entries.
+# Default is not to hide obsolete structures
+#    hide_obsolete : False
+
   - name : photon
     engine : photon
     shortcut : ph
@@ -377,7 +396,7 @@ engines:
     timeout : 10.0
     disabled : True
 
-  - name : scanr_structures
+  - name : scanr structures
     shortcut: scs
     engine : scanr_structures
     disabled : True
@@ -495,6 +514,12 @@ engines:
     timeout: 6.0
     categories : science
 
+  - name : seedpeer
+    engine : seedpeer
+    shortcut: speu
+    categories: files, music, videos
+    disabled: True
+
   - name : dictzone
     engine : dictzone
     shortcut : dc

searx/settings_robot.yml

@@ -15,7 +15,7 @@ server:
 
 ui:
     themes_path : ""
-    default_theme : default
+    default_theme : legacy
     default_locale : ""
 
 outgoing:
@@ -23,12 +23,12 @@ outgoing:
     useragent_suffix : ""
 
 engines:
-  - name : general_dummy
+  - name : general dummy
     engine : dummy
     categories : general
     shortcut : gd
 
-  - name : dummy_dummy
+  - name : dummy dummy
     engine : dummy
     categories : dummy
     shortcut : dd

searx/templates/courgette/base.html

@@ -22,7 +22,7 @@
         {% endblock %}
         {% block meta %}{% endblock %}
         {% block head %}
-        <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+        <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
         {% endblock %}
         <script type="text/javascript">
             searx = {};

searx/templates/default/404.html → searx/templates/legacy/404.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block content %}
 <div class="center">
     <h1>{{ _('Page not found') }}</h1>

searx/templates/default/about.html → searx/templates/legacy/about.html

@@ -1,6 +1,6 @@
-{% extends 'default/base.html' %}
+{% extends 'legacy/base.html' %}
 {% block content %}
-{% include 'default/github_ribbon.html' %}
+{% include 'legacy/github_ribbon.html' %}
 <div class="row"{% if rtl %} dir="ltr"{% endif %}>
     <h1>About <a href="{{ url_for('index') }}">searx</a></h1>
 

searx/templates/default/base.html → searx/templates/legacy/base.html

@@ -17,7 +17,7 @@
         {% endblock %}
         {% block meta %}{% endblock %}
         {% block head %}
-        <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+        <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
         {% endblock %}
     </head>
     <body>

searx/templates/default/index.html → searx/templates/legacy/index.html

@@ -1,8 +1,8 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block content %}
 <div class="center">
     <div class="title"><h1>searx</h1></div>
-    {% include 'default/search.html' %}
+    {% include 'legacy/search.html' %}
     <p class="top_margin">
     	{% if rtl %}
     	<a href="{{ url_for('preferences') }}" class="hmarg">{{ _('preferences') }}</a>
@@ -13,6 +13,6 @@
         {% endif %}
     </p>
 </div>
-{% include 'default/github_ribbon.html' %}
+{% include 'legacy/github_ribbon.html' %}
 {% endblock %}
 

searx/templates/default/preferences.html → searx/templates/legacy/preferences.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <div class="row">
@@ -8,7 +8,7 @@
     <fieldset>
         <legend>{{ _('Default categories') }}</legend>
         {% set display_tooltip = false %}
-        {% include 'default/categories.html' %}
+        {% include 'legacy/categories.html' %}
     </fieldset>
     <fieldset>
         <legend>{{ _('Search language') }}</legend>

searx/templates/default/results.html → searx/templates/legacy/results.html

@@ -1,10 +1,10 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block title %}{{ q }} - {% endblock %}
 {% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
 {% block content %}
 <div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>
 <div class="small search center">
-    {% include 'default/search.html' %}
+    {% include 'legacy/search.html' %}
 </div>
 <div id="results">
     <div id="sidebar">
@@ -55,16 +55,16 @@
     {% if infoboxes %}
     <div id="infoboxes">
       {% for infobox in infoboxes %}
-         {% include 'default/infobox.html' %}
+         {% include 'legacy/infobox.html' %}
       {% endfor %}
     </div>
     {% endif %}
 
     {% for result in results %}
         {% if result['template'] %}
-            {% include get_result_template('default', result['template']) %}
+            {% include get_result_template('legacy', result['template']) %}
         {% else %}
-            {% include 'default/result_templates/default.html' %}
+            {% include 'legacy/result_templates/default.html' %}
         {% endif %}
     {% endfor %}
 

searx/templates/default/search.html → searx/templates/legacy/search.html

@@ -4,5 +4,5 @@
         <input type="submit" value="search" id="search_submit" />
     </div>
     {% set display_tooltip = true %}
-    {% include 'default/categories.html' %}
+    {% include 'legacy/categories.html' %}
 </form>

searx/templates/default/stats.html → searx/templates/legacy/stats.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <h2>{{ _('Engine stats') }}</h2>

searx/templates/oscar/macros.html

@@ -33,6 +33,9 @@
         <span class="label label-default">{{ engine }}</span>
     {% endfor %}
     <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+    {% if proxify %}
+    <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+    {% endif %}
 </div>
     <div class="text-muted"><small>{{ result.pretty_url }}</small></div>
 {%- endmacro %}
@@ -44,6 +47,9 @@
         <span class="label label-default">{{ engine }}</span>
     {% endfor %}
     <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+    {% if proxify %}
+    <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+    {% endif %}
     <div class="text-muted"><small>{{ result.pretty_url }}</small></div>
 {%- endmacro %}
 

searx/templates/pix-art/preferences.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <div class="row">

searx/templates/pix-art/stats.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <h2>{{ _('Engine stats') }}</h2>

searx/utils.py

@@ -252,12 +252,27 @@ def get_torrent_size(filesize, filesize_multiplier):
             filesize = int(filesize * 1024 * 1024)
         elif filesize_multiplier == 'KB':
             filesize = int(filesize * 1024)
+        elif filesize_multiplier == 'TiB':
+            filesize = int(filesize * 1000 * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'GiB':
+            filesize = int(filesize * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'MiB':
+            filesize = int(filesize * 1000 * 1000)
+        elif filesize_multiplier == 'KiB':
+            filesize = int(filesize * 1000)
     except:
         filesize = None
 
     return filesize
 
 
+def convert_str_to_int(number_str):
+    if number_str.isdigit():
+        return int(number_str)
+    else:
+        return 0
+
+
 def is_valid_lang(lang):
     is_abbr = (len(lang) == 2)
     if is_abbr:

searx/webapp.py

@@ -22,10 +22,11 @@ if __name__ == '__main__':
     from os.path import realpath, dirname
     path.append(realpath(dirname(realpath(__file__)) + '/../'))
 
-import json
 import cStringIO
-import os
 import hashlib
+import hmac
+import json
+import os
 import requests
 
 from searx import logger
@@ -245,6 +246,20 @@ def url_for_theme(endpoint, override_theme=None, **values):
     return url_for(endpoint, **values)
 
 
+def proxify(url):
+    if url.startswith('//'):
+        url = 'https:' + url
+
+    if not settings.get('result_proxy'):
+        return url
+
+    h = hmac.new(settings['result_proxy']['key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
+
+    return '{0}?{1}'.format(settings['result_proxy']['url'],
+                            urlencode(dict(mortyurl=url.encode('utf-8'),
+                                           mortyhash=h)))
+
+
 def image_proxify(url):
 
     if url.startswith('//'):
@@ -253,8 +268,7 @@ def image_proxify(url):
     if not request.preferences.get_value('image_proxy'):
         return url
 
-    hash_string = url + settings['server']['secret_key']
-    h = hashlib.sha256(hash_string.encode('utf-8')).hexdigest()
+    h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
 
     return '{0}?{1}'.format(url_for('image_proxy'),
                             urlencode(dict(url=url.encode('utf-8'), h=h)))
@@ -313,6 +327,8 @@ def render(template_name, override_theme=None, **kwargs):
 
     kwargs['image_proxify'] = image_proxify
 
+    kwargs['proxify'] = proxify if settings.get('result_proxy') else None
+
     kwargs['get_result_template'] = get_result_template
 
     kwargs['theme'] = get_current_theme_name(override=override_theme)
@@ -602,7 +618,7 @@ def image_proxy():
     if not url:
         return '', 400
 
-    h = hashlib.sha256(url + settings['server']['secret_key'].encode('utf-8')).hexdigest()
+    h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest()
 
     if h != request.args.get('h'):
         return '', 400
@@ -660,6 +676,7 @@ Allow: /
 Allow: /about
 Disallow: /stats
 Disallow: /preferences
+Disallow: /*?*q=*
 """, mimetype='text/plain')
 
 

tests/robot/test_basic.robot

@@ -4,6 +4,14 @@ Test Setup      Open Browser  http://localhost:11111/
 Test Teardown   Close All Browsers
 
 
+*** Keywords ***
+Submit Preferences
+    Set Selenium Speed  2 seconds
+    Submit Form  id=search_form
+    Location Should Be  http://localhost:11111/
+    Set Selenium Speed  0 seconds
+
+
 *** Test Cases ***
 Front page
     Page Should Contain  about
@@ -24,8 +32,8 @@ Preferences page
     Page Should Contain  Preferences
     Page Should Contain  Default categories
     Page Should Contain  Currently used search engines
-    Page Should Contain  dummy_dummy
-    Page Should Contain  general_dummy
+    Page Should Contain  dummy dummy
+    Page Should Contain  general dummy
 
 Switch category
     Go To  http://localhost:11111/preferences
@@ -33,8 +41,7 @@ Switch category
     Page Should Contain Checkbox  category_dummy
     Click Element  xpath=//*[.="general"]
     Click Element  xpath=//*[.="dummy"]
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Checkbox Should Not Be Selected  category_general
     Checkbox Should Be Selected  category_dummy
 
@@ -43,8 +50,7 @@ Change language
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
     Select From List  locale  hu
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Page Should Contain  rólunk
     Page Should Contain  beállítások
 
@@ -53,13 +59,11 @@ Change method
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
     Select From List  method  GET
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  method  GET
     Select From List  method  POST
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  method  POST
 
@@ -67,10 +71,9 @@ Change theme
     Page Should Contain  about
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
-    List Selection Should Be  theme  default
+    List Selection Should Be  theme  legacy
     Select From List  theme  oscar
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  theme  oscar
 
@@ -80,8 +83,7 @@ Change safesearch
     Go To  http://localhost:11111/preferences
     List Selection Should Be  safesearch  None
     Select From List  safesearch  Strict
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  safesearch  Strict
 
@@ -91,8 +93,7 @@ Change image proxy
     Go To  http://localhost:11111/preferences
     List Selection Should Be  image_proxy  Disabled
     Select From List  image_proxy  Enabled
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  image_proxy  Enabled
 
@@ -102,8 +103,7 @@ Change search language
     Go To  http://localhost:11111/preferences
     List Selection Should Be  language  Automatic
     Select From List  language  Turkish (Turkey) - tr_TR
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  language  Turkish (Turkey) - tr_TR
 
@@ -113,8 +113,7 @@ Change autocomplete
     Go To  http://localhost:11111/preferences
     List Selection Should Be  autocomplete  -
     Select From List  autocomplete  google
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  autocomplete  google
 
@@ -126,8 +125,7 @@ Change allowed/disabled engines
     Element Should Contain  xpath=//label[@class="deny"][@for='engine_dummy_dummy_dummy']  Block
     Element Should Contain  xpath=//label[@class="deny"][@for='engine_general_general_dummy']  Block
     Click Element  xpath=//label[@class="deny"][@for='engine_general_general_dummy']
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Page Should Contain  about
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
@@ -139,18 +137,16 @@ Block a plugin
     Page Should Contain  about
     Page Should Contain  preferences
     Go To  http://localhost:11111/preferences
-    List Selection Should Be  theme  default
+    List Selection Should Be  theme  legacy
     Select From List  theme  oscar
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     List Selection Should Be  theme  oscar
     Page Should Contain  Plugins
     Click Link  Plugins
     Checkbox Should Not Be Selected  id=plugin_HTTPS_rewrite
     Click Element  xpath=//label[@for='plugin_HTTPS_rewrite']
-    Submit Form  id=search_form
-    Location Should Be  http://localhost:11111/
+    Submit Preferences
     Go To  http://localhost:11111/preferences
     Page Should Contain  Plugins
     Click Link  Plugins

tests/unit/engines/test_digbt.py

@@ -28,7 +28,9 @@ class TestDigBTEngine(SearxTestCase):
         <table class="table">
             <tr><td class="x-item">
             <div>
-                <a title="The Big Bang Theory" class="title" href="/The-Big-Bang-Theory-d2.html">The Big Bang Theory</a>
+                <a title="The Big Bang Theory" class="title" href="/The-Big-Bang-Theory-d2.html">
+                    The Big <span class="highlight">Bang</span> Theory
+                </a>
                 <span class="ctime"><span style="color:red;">4 hours ago</span></span>
             </div>
             <div class="files">

tests/unit/engines/test_kickass.py

@@ -14,7 +14,7 @@ class TestKickassEngine(SearxTestCase):
         params = kickass.request(query, dicto)
         self.assertIn('url', params)
         self.assertIn(query, params['url'])
-        self.assertIn('kickass.to', params['url'])
+        self.assertIn('kickass.cd', params['url'])
         self.assertFalse(params['verify'])
 
     def test_response(self):
@@ -84,7 +84,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">449 <span>bytes</span></td>
+                <td class="nobr center">449 bytes</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">10</td>
@@ -97,7 +97,7 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 1)
         self.assertEqual(results[0]['title'], 'This should be the title')
-        self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+        self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
         self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
@@ -191,7 +191,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>KB</span></td>
+                <td class="nobr center">1 KiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">10</td>
@@ -235,7 +235,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>MB</span></td>
+                <td class="nobr center">1 MiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">9</td>
@@ -279,7 +279,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>GB</span></td>
+                <td class="nobr center">1 GiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">8</td>
@@ -323,7 +323,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">1 <span>TB</span></td>
+                <td class="nobr center">1 TiB</td>
                 <td class="center">4</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">7</td>
@@ -367,7 +367,7 @@ class TestKickassEngine(SearxTestCase):
                         </span>
                     </div>
                 </td>
-                <td class="nobr center">z <span>bytes</span></td>
+                <td class="nobr center">z bytes</td>
                 <td class="center">r</td>
                 <td class="center">2&nbsp;years</td>
                 <td class="green center">a</td>
@@ -380,17 +380,17 @@ class TestKickassEngine(SearxTestCase):
         self.assertEqual(type(results), list)
         self.assertEqual(len(results), 5)
         self.assertEqual(results[0]['title'], 'This should be the title')
-        self.assertEqual(results[0]['url'], 'https://kickass.to/url.html')
+        self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')
         self.assertEqual(results[0]['content'], 'Posted by riri in Other &gt; Unsorted')
         self.assertEqual(results[0]['seed'], 10)
         self.assertEqual(results[0]['leech'], 1)
         self.assertEqual(results[0]['files'], 4)
         self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:MAGNETURL&dn=test')
         self.assertEqual(results[0]['torrentfile'], 'http://torcache.net/torrent/53917.torrent?title=test')
-        self.assertEqual(results[0]['filesize'], 1024)
-        self.assertEqual(results[1]['filesize'], 1048576)
-        self.assertEqual(results[2]['filesize'], 1073741824)
-        self.assertEqual(results[3]['filesize'], 1099511627776)
+        self.assertEqual(results[0]['filesize'], 1000)
+        self.assertEqual(results[1]['filesize'], 1000000)
+        self.assertEqual(results[2]['filesize'], 1000000000)
+        self.assertEqual(results[3]['filesize'], 1000000000000)
         self.assertEqual(results[4]['seed'], 0)
         self.assertEqual(results[4]['leech'], 0)
         self.assertEqual(results[4]['files'], None)

tests/unit/engines/test_pdbe.py

@@ -0,0 +1,109 @@
+import mock
+from collections import defaultdict
+from searx.engines import pdbe
+from searx.testing import SearxTestCase
+
+
+class TestPdbeEngine(SearxTestCase):
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        params = pdbe.request(query, dicto)
+        self.assertTrue('url' in params)
+        self.assertTrue('ebi.ac.uk' in params['url'])
+        self.assertTrue('data' in params)
+        self.assertTrue('q' in params['data'])
+        self.assertTrue(query in params['data']['q'])
+        self.assertTrue('wt' in params['data'])
+        self.assertTrue('json' in params['data']['wt'])
+        self.assertTrue('method' in params)
+        self.assertTrue(params['method'] == 'POST')
+
+    def test_response(self):
+        self.assertRaises(AttributeError, pdbe.response, None)
+        self.assertRaises(AttributeError, pdbe.response, [])
+        self.assertRaises(AttributeError, pdbe.response, '')
+        self.assertRaises(AttributeError, pdbe.response, '[]')
+
+        json = """
+{
+  "response": {
+    "docs": [
+      {
+        "citation_title": "X-ray crystal structure of ferric Aplysia limacina myoglobin in different liganded states.",
+        "citation_year": 1993,
+        "entry_author_list": [
+          "Conti E, Moser C, Rizzi M, Mattevi A, Lionetti C, Coda A, Ascenzi P, Brunori M, Bolognesi M"
+        ],
+        "journal": "J. Mol. Biol.",
+        "journal_page": "498-508",
+        "journal_volume": "233",
+        "pdb_id": "2fal",
+        "status": "REL",
+        "title": "X-RAY CRYSTAL STRUCTURE OF FERRIC APLYSIA LIMACINA MYOGLOBIN IN DIFFERENT LIGANDED STATES"
+      }
+    ],
+    "numFound": 1,
+    "start": 0
+  },
+  "responseHeader": {
+    "QTime": 0,
+    "params": {
+      "q": "2fal",
+      "wt": "json"
+    },
+    "status": 0
+  }
+}
+"""
+
+        response = mock.Mock(text=json)
+        results = pdbe.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['title'],
+                         'X-RAY CRYSTAL STRUCTURE OF FERRIC APLYSIA LIMACINA MYOGLOBIN IN DIFFERENT LIGANDED STATES')
+        self.assertEqual(results[0]['url'], pdbe.pdbe_entry_url.format(pdb_id='2fal'))
+        self.assertEqual(results[0]['img_src'], pdbe.pdbe_preview_url.format(pdb_id='2fal'))
+        self.assertTrue('Conti E' in results[0]['content'])
+        self.assertTrue('X-ray crystal structure of ferric Aplysia limacina myoglobin in different liganded states.' in
+                        results[0]['content'])
+        self.assertTrue('1993' in results[0]['content'])
+
+        # Testing proper handling of PDB entries marked as obsolete
+        json = """
+{
+  "response": {
+    "docs": [
+      {
+        "citation_title": "Obsolete entry test",
+        "citation_year": 2016,
+        "entry_author_list": ["Doe J"],
+        "journal": "J. Obs.",
+        "journal_page": "1-2",
+        "journal_volume": "1",
+        "pdb_id": "xxxx",
+        "status": "OBS",
+        "title": "OBSOLETE ENTRY TEST",
+        "superseded_by": "yyyy"
+      }
+    ],
+    "numFound": 1,
+    "start": 0
+  },
+  "responseHeader": {
+    "QTime": 0,
+    "params": {
+      "q": "xxxx",
+      "wt": "json"
+    },
+    "status": 0
+  }
+}
+"""
+        response = mock.Mock(text=json)
+        results = pdbe.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['title'], 'OBSOLETE ENTRY TEST (OBSOLETE)')
+        self.assertTrue(results[0]['content'].startswith('<em>This entry has been superseded by'))

tests/unit/engines/test_seedpeer.py

@@ -0,0 +1,51 @@
+import mock
+from collections import defaultdict
+from searx.engines import seedpeer
+from searx.testing import SearxTestCase
+from datetime import datetime
+
+
+class TestSeedPeerEngine(SearxTestCase):
+
+    html = ''
+    with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
+        html += fixture.read()
+
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['pageno'] = 1
+        params = seedpeer.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertIn(query, params['url'])
+        self.assertIn('seedpeer.eu', params['url'])
+
+    def test_response_raises_attr_error_on_empty_response(self):
+        self.assertRaises(AttributeError, seedpeer.response, None)
+        self.assertRaises(AttributeError, seedpeer.response, [])
+        self.assertRaises(AttributeError, seedpeer.response, '')
+        self.assertRaises(AttributeError, seedpeer.response, '[]')
+
+    def test_response_returns_empty_list(self):
+        response = mock.Mock(text='<html></html>')
+        self.assertEqual(seedpeer.response(response), [])
+
+    def test_response_returns_all_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertTrue(isinstance(results, list))
+        self.assertEqual(len(results), 2)
+
+    def test_response_returns_correct_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertEqual(
+            results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
+        )
+        self.assertEqual(
+            results[0]['url'],
+            'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
+        )
+        self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
+        self.assertEqual(results[0]['seed'], '861')
+        self.assertEqual(results[0]['leech'], '332')

tests/unit/test_webapp.py

@@ -44,7 +44,7 @@ class ViewsTestCase(SearxTestCase):
         webapp.Search.search = search_mock
 
         def get_current_theme_name_mock(override=None):
-            return 'default'
+            return 'legacy'
 
         webapp.get_current_theme_name = get_current_theme_name_mock
 
@@ -58,7 +58,7 @@ class ViewsTestCase(SearxTestCase):
     def test_index_html(self):
         result = self.app.post('/', data={'q': 'test'})
         self.assertIn(
-            '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>',  # noqa
+            '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/legacy/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>',  # noqa
             result.data
         )
         self.assertIn(