Merge branch 'master' of https://github.com/asciimoo/searx into boilerplate

Dockerfile

@@ -1,10 +1,17 @@
 FROM alpine:3.10
+ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]
+EXPOSE 8080
+VOLUME /etc/searx
+VOLUME /var/log/uwsgi
 
-ARG VERSION_GITCOMMIT=unknow
-ARG SEARX_GIT_VERSION=unknow
+ARG VERSION_GITCOMMIT=unknown
+ARG SEARX_GIT_VERSION=unknown
 
-ARG SEARX_GID=1000
-ARG SEARX_UID=1000
+ARG SEARX_GID=977
+ARG SEARX_UID=977
+
+RUN addgroup -g ${SEARX_GID} searx && \
+    adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx
 
 ARG TIMESTAMP_SETTINGS=0
 ARG TIMESTAMP_UWSGI=0
@@ -16,19 +23,14 @@ ENV INSTANCE_NAME=searx \
     BASE_URL= \
     MORTY_KEY= \
     MORTY_URL=
-EXPOSE 8080
-VOLUME /etc/searx
-VOLUME /var/log/uwsgi
 
 WORKDIR /usr/local/searx
 
-RUN addgroup -g ${SEARX_GID} searx && \
-    adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx
 
 COPY requirements.txt ./requirements.txt
 
-RUN apk -U upgrade \
- && apk add -t build-dependencies \
+RUN apk upgrade --no-cache \
+ && apk add --no-cache -t build-dependencies \
     build-base \
     py3-setuptools \
     python3-dev \
@@ -38,7 +40,7 @@ RUN apk -U upgrade \
     openssl-dev \
     tar \
     git \
- && apk add \
+ && apk add --no-cache \
     ca-certificates \
     su-exec \
     python3 \
@@ -50,8 +52,7 @@ RUN apk -U upgrade \
     uwsgi-python3 \
  && pip3 install --upgrade pip \
  && pip3 install --no-cache -r requirements.txt \
- && apk del build-dependencies \
- && rm -f /var/cache/apk/*
+ && apk del build-dependencies
 
 COPY --chown=searx:searx . .
 
@@ -62,7 +63,6 @@ RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
       echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \
     fi
 
-ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]
 
 # Keep this argument at the end since it change each time
 ARG LABEL_DATE=

searx/engines/bing.py

@@ -18,7 +18,7 @@ from lxml import html
 from searx import logger, utils
 from searx.engines.xpath import extract_text
 from searx.url_utils import urlencode
-from searx.utils import match_language, gen_useragent
+from searx.utils import match_language, gen_useragent, eval_xpath
 
 logger = logger.getChild('bing engine')
 
@@ -65,11 +65,11 @@ def response(resp):
 
     dom = html.fromstring(resp.text)
     # parse results
-    for result in dom.xpath('//div[@class="sa_cc"]'):
-        link = result.xpath('.//h3/a')[0]
+    for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
+        link = eval_xpath(result, './/h3/a')[0]
         url = link.attrib.get('href')
         title = extract_text(link)
-        content = extract_text(result.xpath('.//p'))
+        content = extract_text(eval_xpath(result, './/p'))
 
         # append result
         results.append({'url': url,
@@ -77,11 +77,11 @@ def response(resp):
                         'content': content})
 
     # parse results again if nothing is found yet
-    for result in dom.xpath('//li[@class="b_algo"]'):
-        link = result.xpath('.//h2/a')[0]
+    for result in eval_xpath(dom, '//li[@class="b_algo"]'):
+        link = eval_xpath(result, './/h2/a')[0]
         url = link.attrib.get('href')
         title = extract_text(link)
-        content = extract_text(result.xpath('.//p'))
+        content = extract_text(eval_xpath(result, './/p'))
 
         # append result
         results.append({'url': url,
@@ -89,7 +89,7 @@ def response(resp):
                         'content': content})
 
     try:
-        result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))
+        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))
         result_len_container = utils.to_string(result_len_container)
         if "-" in result_len_container:
             # Remove the part "from-to" for paginated request ...
@@ -113,9 +113,9 @@ def response(resp):
 def _fetch_supported_languages(resp):
     supported_languages = []
     dom = html.fromstring(resp.text)
-    options = dom.xpath('//div[@id="limit-languages"]//input')
+    options = eval_xpath(dom, '//div[@id="limit-languages"]//input')
     for option in options:
-        code = option.xpath('./@id')[0].replace('_', '-')
+        code = eval_xpath(option, './@id')[0].replace('_', '-')
         if code == 'nb':
             code = 'no'
         supported_languages.append(code)

searx/engines/dictzone.py

@@ -11,7 +11,7 @@
 
 import re
 from lxml import html
-from searx.utils import is_valid_lang
+from searx.utils import is_valid_lang, eval_xpath
 from searx.url_utils import urljoin
 
 categories = ['general']
@@ -47,14 +47,14 @@ def response(resp):
 
     dom = html.fromstring(resp.text)
 
-    for k, result in enumerate(dom.xpath(results_xpath)[1:]):
+    for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]):
         try:
-            from_result, to_results_raw = result.xpath('./td')
+            from_result, to_results_raw = eval_xpath(result, './td')
         except:
             continue
 
         to_results = []
-        for to_result in to_results_raw.xpath('./p/a'):
+        for to_result in eval_xpath(to_results_raw, './p/a'):
             t = to_result.text_content()
             if t.strip():
                 to_results.append(to_result.text_content())

searx/engines/doku.py

@@ -11,6 +11,7 @@
 
 from lxml.html import fromstring
 from searx.engines.xpath import extract_text
+from searx.utils import eval_xpath
 from searx.url_utils import urlencode
 
 # engine dependent config
@@ -45,16 +46,16 @@ def response(resp):
 
     # parse results
     # Quickhits
-    for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'):
+    for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'):
         try:
-            res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
+            res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]
         except:
             continue
 
         if not res_url:
             continue
 
-        title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
+        title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))
 
         # append result
         results.append({'title': title,
@@ -62,13 +63,13 @@ def response(resp):
                         'url': base_url + res_url})
 
     # Search results
-    for r in doc.xpath('//dl[@class="search_results"]/*'):
+    for r in eval_xpath(doc, '//dl[@class="search_results"]/*'):
         try:
             if r.tag == "dt":
-                res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
-                title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
+                res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]
+                title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))
             elif r.tag == "dd":
-                content = extract_text(r.xpath('.'))
+                content = extract_text(eval_xpath(r, '.'))
 
                 # append result
                 results.append({'title': title,

searx/engines/duckduckgo.py

@@ -18,7 +18,7 @@ from json import loads
 from searx.engines.xpath import extract_text
 from searx.poolrequests import get
 from searx.url_utils import urlencode
-from searx.utils import match_language
+from searx.utils import match_language, eval_xpath
 
 # engine dependent config
 categories = ['general']
@@ -106,19 +106,19 @@ def response(resp):
     doc = fromstring(resp.text)
 
     # parse results
-    for i, r in enumerate(doc.xpath(result_xpath)):
+    for i, r in enumerate(eval_xpath(doc, result_xpath)):
         if i >= 30:
             break
         try:
-            res_url = r.xpath(url_xpath)[-1]
+            res_url = eval_xpath(r, url_xpath)[-1]
         except:
             continue
 
         if not res_url:
             continue
 
-        title = extract_text(r.xpath(title_xpath))
-        content = extract_text(r.xpath(content_xpath))
+        title = extract_text(eval_xpath(r, title_xpath))
+        content = extract_text(eval_xpath(r, content_xpath))
 
         # append result
         results.append({'title': title,

searx/engines/duckduckgo_definitions.py

@@ -1,3 +1,14 @@
+"""
+DuckDuckGo (definitions)
+
+- `Instant Answer API`_
+- `DuckDuckGo query`_
+
+.. _Instant Answer API: https://duckduckgo.com/api
+.. _DuckDuckGo query: https://api.duckduckgo.com/?q=DuckDuckGo&format=json&pretty=1
+
+"""
+
 import json
 from lxml import html
 from re import compile
@@ -25,7 +36,8 @@ def result_to_text(url, text, htmlResult):
 def request(query, params):
     params['url'] = url.format(query=urlencode({'q': query}))
     language = match_language(params['language'], supported_languages, language_aliases)
-    params['headers']['Accept-Language'] = language.split('-')[0]
+    language = language.split('-')[0]
+    params['headers']['Accept-Language'] = language
     return params
 
 
@@ -43,8 +55,9 @@ def response(resp):
 
     # add answer if there is one
     answer = search_res.get('Answer', '')
-    if answer != '':
-        results.append({'answer': html_to_text(answer)})
+    if answer:
+        if search_res.get('AnswerType', '') not in ['calc']:
+            results.append({'answer': html_to_text(answer)})
 
     # add infobox
     if 'Definition' in search_res:

searx/engines/duden.py

@@ -11,6 +11,7 @@
 from lxml import html, etree
 import re
 from searx.engines.xpath import extract_text
+from searx.utils import eval_xpath
 from searx.url_utils import quote, urljoin
 from searx import logger
 
@@ -52,9 +53,9 @@ def response(resp):
     dom = html.fromstring(resp.text)
 
     try:
-        number_of_results_string = re.sub('[^0-9]', '', dom.xpath(
-            '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]
-        )
+        number_of_results_string =\
+            re.sub('[^0-9]', '',
+                   eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0])
 
         results.append({'number_of_results': int(number_of_results_string)})
 
@@ -62,12 +63,12 @@ def response(resp):
         logger.debug("Couldn't read number of results.")
         pass
 
-    for result in dom.xpath('//section[not(contains(@class, "essay"))]'):
+    for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'):
         try:
-            url = result.xpath('.//h2/a')[0].get('href')
+            url = eval_xpath(result, './/h2/a')[0].get('href')
             url = urljoin(base_url, url)
-            title = result.xpath('string(.//h2/a)').strip()
-            content = extract_text(result.xpath('.//p'))
+            title = eval_xpath(result, 'string(.//h2/a)').strip()
+            content = extract_text(eval_xpath(result, './/p'))
             # append result
             results.append({'url': url,
                             'title': title,

searx/engines/gigablast.py

@@ -15,6 +15,7 @@ from json import loads
 from time import time
 from lxml.html import fromstring
 from searx.url_utils import urlencode
+from searx.utils import eval_xpath
 
 # engine dependent config
 categories = ['general']
@@ -99,9 +100,9 @@ def response(resp):
 def _fetch_supported_languages(resp):
     supported_languages = []
     dom = fromstring(resp.text)
-    links = dom.xpath('//span[@id="menu2"]/a')
+    links = eval_xpath(dom, '//span[@id="menu2"]/a')
     for link in links:
-        href = link.xpath('./@href')[0].split('lang%3A')
+        href = eval_xpath(link, './@href')[0].split('lang%3A')
         if len(href) == 2:
             code = href[1].split('_')
             if len(code) == 2:

searx/engines/google.py

@@ -14,7 +14,7 @@ from lxml import html, etree
 from searx.engines.xpath import extract_text, extract_url
 from searx import logger
 from searx.url_utils import urlencode, urlparse, parse_qsl
-from searx.utils import match_language
+from searx.utils import match_language, eval_xpath
 
 logger = logger.getChild('google engine')
 
@@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname):
 
 # returns extract_text on the first result selected by the xpath or None
 def extract_text_from_dom(result, xpath):
-    r = result.xpath(xpath)
+    r = eval_xpath(result, xpath)
     if len(r) > 0:
         return extract_text(r[0])
     return None
@@ -227,21 +227,21 @@ def response(resp):
     # convert the text to dom
     dom = html.fromstring(resp.text)
 
-    instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
+    instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')
     if instant_answer:
         results.append({'answer': u' '.join(instant_answer)})
     try:
-        results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
+        results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]
                           .split()[1].replace(',', ''))
         results.append({'number_of_results': results_num})
     except:
         pass
 
     # parse results
-    for result in dom.xpath(results_xpath):
+    for result in eval_xpath(dom, results_xpath):
         try:
-            title = extract_text(result.xpath(title_xpath)[0])
-            url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)
+            title = extract_text(eval_xpath(result, title_xpath)[0])
+            url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)
             parsed_url = urlparse(url, google_hostname)
 
             # map result
@@ -250,7 +250,7 @@ def response(resp):
                 continue
                 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
                 #     print "yooooo"*30
-                #     x = result.xpath(map_near)
+                #     x = eval_xpath(result, map_near)
                 #     if len(x) > 0:
                 #         # map : near the location
                 #         results = results + parse_map_near(parsed_url, x, google_hostname)
@@ -287,11 +287,11 @@ def response(resp):
             continue
 
     # parse suggestion
-    for suggestion in dom.xpath(suggestion_xpath):
+    for suggestion in eval_xpath(dom, suggestion_xpath):
         # append suggestion
         results.append({'suggestion': extract_text(suggestion)})
 
-    for correction in dom.xpath(spelling_suggestion_xpath):
+    for correction in eval_xpath(dom, spelling_suggestion_xpath):
         results.append({'correction': extract_text(correction)})
 
     # return results
@@ -300,9 +300,9 @@ def response(resp):
 
 def parse_images(result, google_hostname):
     results = []
-    for image in result.xpath(images_xpath):
-        url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)
-        img_src = extract_text(image.xpath(image_img_src_xpath)[0])
+    for image in eval_xpath(result, images_xpath):
+        url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)
+        img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])
 
         # append result
         results.append({'url': url,
@@ -389,10 +389,10 @@ def attributes_to_html(attributes):
 def _fetch_supported_languages(resp):
     supported_languages = {}
     dom = html.fromstring(resp.text)
-    options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')
+    options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')
     for option in options:
-        code = option.xpath('./@value')[0].split('_')[-1]
-        name = option.xpath('./@data-name')[0].title()
+        code = eval_xpath(option, './@value')[0].split('_')[-1]
+        name = eval_xpath(option, './@data-name')[0].title()
         supported_languages[code] = {"name": name}
 
     return supported_languages

searx/engines/seedpeer.py

@@ -0,0 +1,78 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     https://seedpeer.me
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from lxml import html
+from json import loads
+from operator import itemgetter
+from searx.url_utils import quote, urljoin
+from searx.engines.xpath import extract_text
+
+
+url = 'https://seedpeer.me/'
+search_url = url + 'search/{search_term}?page={page_no}'
+torrent_file_url = url + 'torrent/{torrent_hash}'
+
+# specific xpath variables
+script_xpath = '//script[@type="text/javascript"][not(@src)]'
+torrent_xpath = '(//table)[2]/tbody/tr'
+link_xpath = '(./td)[1]/a/@href'
+age_xpath = '(./td)[2]'
+size_xpath = '(./td)[3]'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'])
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    result_rows = dom.xpath(torrent_xpath)
+
+    try:
+        script_element = dom.xpath(script_xpath)[0]
+        json_string = script_element.text[script_element.text.find('{'):]
+        torrents_json = loads(json_string)
+    except:
+        return []
+
+    # parse results
+    for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']):
+        title = torrent_json['name']
+        seed = int(torrent_json['seeds'])
+        leech = int(torrent_json['peers'])
+        size = int(torrent_json['size'])
+        torrent_hash = torrent_json['hash']
+
+        torrentfile = torrent_file_url.format(torrent_hash=torrent_hash)
+        magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash)
+
+        age = extract_text(torrent_row.xpath(age_xpath))
+        link = torrent_row.xpath(link_xpath)[0]
+
+        href = urljoin(url, link)
+
+        # append result
+        results.append({'url': href,
+                        'title': title,
+                        'content': age,
+                        'seed': seed,
+                        'leech': leech,
+                        'filesize': size,
+                        'torrentfile': torrentfile,
+                        'magnetlink': magnetlink,
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)

searx/engines/soundcloud.py

@@ -51,7 +51,9 @@ def get_client_id():
 
     if response.ok:
         tree = html.fromstring(response.content)
-        script_tags = tree.xpath("//script[contains(@src, '/assets/app')]")
+        # script_tags has been moved from /assets/app/ to /assets/ path.  I
+        # found client_id in https://a-v2.sndcdn.com/assets/49-a0c01933-3.js
+        script_tags = tree.xpath("//script[contains(@src, '/assets/')]")
         app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]
 
         # extracts valid app_js urls from soundcloud.com content

searx/engines/startpage.py

@@ -16,6 +16,7 @@ from datetime import datetime, timedelta
 import re
 from searx.engines.xpath import extract_text
 from searx.languages import language_codes
+from searx.utils import eval_xpath
 
 # engine dependent config
 categories = ['general']
@@ -70,8 +71,8 @@ def response(resp):
     dom = html.fromstring(resp.text)
 
     # parse results
-    for result in dom.xpath(results_xpath):
-        links = result.xpath(link_xpath)
+    for result in eval_xpath(dom, results_xpath):
+        links = eval_xpath(result, link_xpath)
         if not links:
             continue
         link = links[0]
@@ -87,8 +88,8 @@ def response(resp):
 
         title = extract_text(link)
 
-        if result.xpath(content_xpath):
-            content = extract_text(result.xpath(content_xpath))
+        if eval_xpath(result, content_xpath):
+            content = extract_text(eval_xpath(result, content_xpath))
         else:
             content = ''
 

searx/engines/wikidata.py

@@ -16,7 +16,7 @@ from searx.poolrequests import get
 from searx.engines.xpath import extract_text
 from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
 from searx.url_utils import urlencode
-from searx.utils import match_language
+from searx.utils import match_language, eval_xpath
 
 from json import loads
 from lxml.html import fromstring
@@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator
 calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
 media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
 
-# xpath_cache
-xpath_cache = {}
-
-
-def get_xpath(xpath_str):
-    result = xpath_cache.get(xpath_str, None)
-    if not result:
-        result = etree.XPath(xpath_str)
-        xpath_cache[xpath_str] = result
-    return result
-
-
-def eval_xpath(element, xpath_str):
-    xpath = get_xpath(xpath_str)
-    return xpath(element)
-
 
 def get_id_cache(result):
     id_cache = {}

searx/engines/xpath.py

@@ -1,6 +1,6 @@
 from lxml import html
 from lxml.etree import _ElementStringResult, _ElementUnicodeResult
-from searx.utils import html_to_text
+from searx.utils import html_to_text, eval_xpath
 from searx.url_utils import unquote, urlencode, urljoin, urlparse
 
 search_url = None
@@ -104,15 +104,15 @@ def response(resp):
     results = []
     dom = html.fromstring(resp.text)
     if results_xpath:
-        for result in dom.xpath(results_xpath):
-            url = extract_url(result.xpath(url_xpath), search_url)
-            title = extract_text(result.xpath(title_xpath))
-            content = extract_text(result.xpath(content_xpath))
+        for result in eval_xpath(dom, results_xpath):
+            url = extract_url(eval_xpath(result, url_xpath), search_url)
+            title = extract_text(eval_xpath(result, title_xpath))
+            content = extract_text(eval_xpath(result, content_xpath))
             tmp_result = {'url': url, 'title': title, 'content': content}
 
             # add thumbnail if available
             if thumbnail_xpath:
-                thumbnail_xpath_result = result.xpath(thumbnail_xpath)
+                thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)
                 if len(thumbnail_xpath_result) > 0:
                     tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)
 
@@ -120,14 +120,14 @@ def response(resp):
     else:
         for url, title, content in zip(
             (extract_url(x, search_url) for
-             x in dom.xpath(url_xpath)),
-            map(extract_text, dom.xpath(title_xpath)),
-            map(extract_text, dom.xpath(content_xpath))
+             x in eval_xpath(dom, url_xpath)),
+            map(extract_text, eval_xpath(dom, title_xpath)),
+            map(extract_text, eval_xpath(dom, content_xpath))
         ):
             results.append({'url': url, 'title': title, 'content': content})
 
     if not suggestion_xpath:
         return results
-    for suggestion in dom.xpath(suggestion_xpath):
+    for suggestion in eval_xpath(dom, suggestion_xpath):
         results.append({'suggestion': extract_text(suggestion)})
     return results

searx/engines/yahoo.py

@@ -14,7 +14,7 @@
 from lxml import html
 from searx.engines.xpath import extract_text, extract_url
 from searx.url_utils import unquote, urlencode
-from searx.utils import match_language
+from searx.utils import match_language, eval_xpath
 
 # engine dependent config
 categories = ['general']
@@ -109,21 +109,21 @@ def response(resp):
     dom = html.fromstring(resp.text)
 
     try:
-        results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0]
+        results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0]
                           .split()[0].replace(',', ''))
         results.append({'number_of_results': results_num})
     except:
         pass
 
     # parse results
-    for result in dom.xpath(results_xpath):
+    for result in eval_xpath(dom, results_xpath):
         try:
-            url = parse_url(extract_url(result.xpath(url_xpath), search_url))
-            title = extract_text(result.xpath(title_xpath)[0])
+            url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url))
+            title = extract_text(eval_xpath(result, title_xpath)[0])
         except:
             continue
 
-        content = extract_text(result.xpath(content_xpath)[0])
+        content = extract_text(eval_xpath(result, content_xpath)[0])
 
         # append result
         results.append({'url': url,
@@ -131,7 +131,7 @@ def response(resp):
                         'content': content})
 
     # if no suggestion found, return results
-    suggestions = dom.xpath(suggestion_xpath)
+    suggestions = eval_xpath(dom, suggestion_xpath)
     if not suggestions:
         return results
 
@@ -148,9 +148,9 @@ def response(resp):
 def _fetch_supported_languages(resp):
     supported_languages = []
     dom = html.fromstring(resp.text)
-    options = dom.xpath('//div[@id="yschlang"]/span/label/input')
+    options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input')
     for option in options:
-        code_parts = option.xpath('./@value')[0][5:].split('_')
+        code_parts = eval_xpath(option, './@value')[0][5:].split('_')
         if len(code_parts) == 2:
             code = code_parts[0] + '-' + code_parts[1].upper()
         else:

searx/results.py

@@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2):
 
         for url2 in infobox2.get('urls', []):
             unique_url = True
-            for url1 in infobox1.get('urls', []):
-                if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))):
+            parsed_url2 = urlparse(url2.get('url', ''))
+            for url1 in urls1:
+                if compare_urls(urlparse(url1.get('url', '')), parsed_url2):
                     unique_url = False
                     break
             if unique_url:
@@ -188,8 +189,9 @@ class ResultContainer(object):
         add_infobox = True
         infobox_id = infobox.get('id', None)
         if infobox_id is not None:
+            parsed_url_infobox_id = urlparse(infobox_id)
             for existingIndex in self.infoboxes:
-                if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)):
+                if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):
                     merge_two_infoboxes(existingIndex, infobox)
                     add_infobox = False
 

searx/settings.yml

@@ -748,6 +748,11 @@ engines:
     page_size : 10
     disabled : True
 
+  - name : seedpeer
+    shortcut : speu
+    engine : seedpeer
+    categories: files, music, videos
+
 #  - name : yacy
 #    engine : yacy
 #    shortcut : ya

searx/templates/courgette/result_templates/torrent.html

@@ -4,7 +4,7 @@
     {% endif %}
     <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>
     {% if result.content %}<span class="content">{{ result.content|safe }}</span><br />{% endif %}
-    {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}
+    {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}
     <span>
         {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} 
         {% if result.torrentfile %}<a href="{{ result.torrentfile }}" class="torrentfile" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('torrent file') }}</a>{% endif %}

searx/templates/legacy/result_templates/torrent.html

@@ -8,6 +8,6 @@
     <p>
         {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} 
         {% if result.torrentfile %}<a href="{{ result.torrentfile }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="torrentfile">{{ _('torrent file') }}</a>{% endif %} - 
-        {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %} 
+        {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %} 
     </p>
 </div>

searx/templates/oscar/result_templates/torrent.html

@@ -3,7 +3,7 @@
 {{ result_header(result, favicons) }}
 {{ result_sub_header(result) }}
 
-{% if result.seed %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}
+{% if result.seed is defined %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}
 {% if result.filesize %}<br />{{ icon('floppy-disk') }} {{ _('Filesize') }} 
     <span class="badge">
         {% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

searx/templates/simple/result_templates/torrent.html

@@ -6,7 +6,7 @@
 {% if result.magnetlink %}<p class="altlink"> &bull; {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}</p>{% endif %}
 {% if result.torrentfile %}<p class="altlink"> &bull; {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}</p>{% endif %}
 
-{% if result.seed %}<p class="stat"> &bull; {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}
+{% if result.seed is defined %}<p class="stat"> &bull; {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> &bull; {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}
 
 {%- if result.filesize %}<p class="stat">{{ icon('floppy-disk') }} {{ _('Filesize') }}<span class="badge">
     {%- if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

searx/utils.py

@@ -13,6 +13,7 @@ from numbers import Number
 from os.path import splitext, join
 from io import open
 from random import choice
+from lxml.etree import XPath
 import sys
 import json
 
@@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)
 useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
                              + "/data/useragents.json", 'r', encoding='utf-8').read())
 
+xpath_cache = dict()
 lang_to_lc_cache = dict()
 
 
@@ -450,3 +452,16 @@ def get_engine_from_settings(name):
             return engine
 
     return {}
+
+
+def get_xpath(xpath_str):
+    result = xpath_cache.get(xpath_str, None)
+    if result is None:
+        result = XPath(xpath_str)
+        xpath_cache[xpath_str] = result
+    return result
+
+
+def eval_xpath(element, xpath_str):
+    xpath = get_xpath(xpath_str)
+    return xpath(element)

searx/webapp.py

@@ -157,20 +157,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None
 
 @babel.localeselector
 def get_locale():
-    locale = request.accept_languages.best_match(settings['locales'].keys())
-
-    if request.preferences.get_value('locale') != '':
-        locale = request.preferences.get_value('locale')
+    if 'locale' in request.form\
+       and request.form['locale'] in settings['locales']:
+        return request.form['locale']
 
     if 'locale' in request.args\
        and request.args['locale'] in settings['locales']:
-        locale = request.args['locale']
+        return request.args['locale']
 
-    if 'locale' in request.form\
-       and request.form['locale'] in settings['locales']:
-        locale = request.form['locale']
+    if request.preferences.get_value('locale') != '':
+        return request.preferences.get_value('locale')
 
-    return locale
+    return request.accept_languages.best_match(settings['locales'].keys())
 
 
 # code-highlighter

tests/unit/engines/test_seedpeer.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from collections import defaultdict
+import mock
+from searx.engines import seedpeer
+from searx.testing import SearxTestCase
+
+
+class TestBtdiggEngine(SearxTestCase):
+
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['pageno'] = 1
+        params = seedpeer.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertIn(query, params['url'])
+        self.assertIn('seedpeer', params['url'])
+
+    def test_response(self):
+        self.assertRaises(AttributeError, seedpeer.response, None)
+        self.assertRaises(AttributeError, seedpeer.response, [])
+        self.assertRaises(AttributeError, seedpeer.response, '')
+        self.assertRaises(AttributeError, seedpeer.response, '[]')
+
+        response = mock.Mock(text='<html></html>')
+        self.assertEqual(seedpeer.response(response), [])
+
+        html = u"""
+        <html>
+        <head>
+            <script></script>
+            <script type="text/javascript" src="not_here.js"></script>
+            <script type="text/javascript">
+                window.initialData=
+                {"data": {"list": [{"name": "Title", "seeds": "10", "peers": "20", "size": "1024", "hash": "abc123"}]}}
+            </script>
+        </head>
+        <body>
+            <table></table>
+            <table>
+                <thead><tr></tr></thead>
+                <tbody>
+                    <tr>
+                        <td><a href="link">Title</a></td>
+                        <td>1 year</td>
+                        <td>1 KB</td>
+                        <td>10</td>
+                        <td>20</td>
+                        <td></td>
+                    </tr>
+                </tbody>
+            </table>
+        </body>
+        </html>
+        """
+        response = mock.Mock(text=html)
+        results = seedpeer.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['title'], 'Title')
+        self.assertEqual(results[0]['url'], 'https://seedpeer.me/link')
+        self.assertEqual(results[0]['seed'], 10)
+        self.assertEqual(results[0]['leech'], 20)
+        self.assertEqual(results[0]['filesize'], 1024)
+        self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123')
+        self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123')