Merge pull request #677 from pydo/feature/seedpeer-engine-integration

Feature/seedpeer engine integration

AUTHORS.rst

@@ -58,3 +58,4 @@ generally made searx better:
 - marc @a01200356
 - Harry Wood @harry-wood
 - Thomas Renard @threnard
+- Pydo `<https://github.com/pydo>`_

searx/engines/seedpeer.py

@@ -0,0 +1,78 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     http://seedpeer.eu
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import quote
+from lxml import html
+from operator import itemgetter
+from searx.engines.xpath import extract_text
+
+
+url = 'http://www.seedpeer.eu/'
+search_url = url + 'search/{search_term}/7/{page_no}.html'
+# specific xpath variables
+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'] - 1)
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    torrent_links = dom.xpath(torrent_xpath)
+    if len(torrent_links) > 0:
+        seeds = dom.xpath(seeds_xpath)
+        peers = dom.xpath(peers_xpath)
+        titles = dom.xpath(title_xpath)
+        sizes = dom.xpath(size_xpath)
+        ages = dom.xpath(age_xpath)
+    else:  # under ~5 results uses a different xpath
+        torrent_links = dom.xpath(alternative_torrent_xpath)
+        seeds = dom.xpath(alternative_seeds_xpath)
+        peers = dom.xpath(alternative_peers_xpath)
+        titles = dom.xpath(alternative_title_xpath)
+        sizes = dom.xpath(alternative_size_xpath)
+        ages = dom.xpath(alternative_age_xpath)
+    # return empty array if nothing is found
+    if not torrent_links:
+        return []
+
+    # parse results
+    for index, result in enumerate(torrent_links):
+        link = result.attrib.get('href')
+        href = urljoin(url, link)
+        results.append({'url': href,
+                        'title': titles[index].text_content(),
+                        'content': '{}, {}'.format(sizes[index], ages[index]),
+                        'seed': seeds[index],
+                        'leech': peers[index],
+
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)

searx/settings.yml

@@ -495,6 +495,12 @@ engines:
     timeout: 6.0
     categories : science
 
+  - name : seedpeer
+    engine : seedpeer
+    shortcut: speu
+    categories: files, music, videos
+    disabled: True
+
   - name : dictzone
     engine : dictzone
     shortcut : dc

tests/unit/engines/test_seedpeer.py

@@ -0,0 +1,51 @@
+import mock
+from collections import defaultdict
+from searx.engines import seedpeer
+from searx.testing import SearxTestCase
+from datetime import datetime
+
+
+class TestSeedPeerEngine(SearxTestCase):
+
+    html = ''
+    with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
+        html += fixture.read()
+
+    def test_request(self):
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['pageno'] = 1
+        params = seedpeer.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertIn(query, params['url'])
+        self.assertIn('seedpeer.eu', params['url'])
+
+    def test_response_raises_attr_error_on_empty_response(self):
+        self.assertRaises(AttributeError, seedpeer.response, None)
+        self.assertRaises(AttributeError, seedpeer.response, [])
+        self.assertRaises(AttributeError, seedpeer.response, '')
+        self.assertRaises(AttributeError, seedpeer.response, '[]')
+
+    def test_response_returns_empty_list(self):
+        response = mock.Mock(text='<html></html>')
+        self.assertEqual(seedpeer.response(response), [])
+
+    def test_response_returns_all_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertTrue(isinstance(results, list))
+        self.assertEqual(len(results), 2)
+
+    def test_response_returns_correct_results(self):
+        response = mock.Mock(text=self.html)
+        results = seedpeer.response(response)
+        self.assertEqual(
+            results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
+        )
+        self.assertEqual(
+            results[0]['url'],
+            'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
+        )
+        self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
+        self.assertEqual(results[0]['seed'], '861')
+        self.assertEqual(results[0]['leech'], '332')