NTab仓库
fjmwkj
/
searxng
mirror of https://github.com/searxng/searxng
1
0
Fork 0
Files Issues 0 Wiki
Browse Source

[fix] fix duden engine (#1594)

rachmadani haryono 5 years ago
parent
87baa74a86
commit
3b1122c5fa
2 changed files with 28 additions and 19 deletions
Unified View Show Diff Stats
  1. 11 8
      searx/engines/duden.py
  2. 17 11
      tests/unit/engines/test_duden.py

+ 11 - 8
searx/engines/duden.py
View File 

@@ -11,7 +11,7 @@
 
 from lxml import html, etree
 
 from lxml import html, etree
 
 import re
 
 import re
 
 from searx.engines.xpath import extract_text
 
 from searx.engines.xpath import extract_text
 
-from searx.url_utils import quote
 
+from searx.url_utils import quote, urljoin
 
 from searx import logger
 
 from searx import logger
 
 
 
 categories = ['general']
 
 categories = ['general']
 
@@ -20,7 +20,7 @@ language_support = False
 
 
 
 # search-url
 
 # search-url
 
 base_url = 'https://www.duden.de/'
 
 base_url = 'https://www.duden.de/'
 
-search_url = base_url + 'suchen/dudenonline/{query}?page={offset}'
 
+search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}'
 
 
 
 
 
 def request(query, params):
 
 def request(query, params):
 
@@ -35,7 +35,11 @@ def request(query, params):
 
     '''
 
     '''
 
 
 
     offset = (params['pageno'] - 1)
 
     offset = (params['pageno'] - 1)
 
-    params['url'] = search_url.format(offset=offset, query=quote(query))
 
+    if offset == 0:
 
+        search_url_fmt = base_url + 'suchen/dudenonline/{query}'
 
+        params['url'] = search_url_fmt.format(query=quote(query))
 
+    else:
 
+        params['url'] = search_url.format(offset=offset, query=quote(query))
 
     return params
 
     return params
 
 
 
 
 
@@ -58,12 +62,11 @@ def response(resp):
 
         logger.debug("Couldn't read number of results.")
 
         logger.debug("Couldn't read number of results.")
 
         pass
 
         pass
 
 
 
-    for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'):
 
+    for result in dom.xpath('//section[not(contains(@class, "essay"))]'):
 
         try:
 
         try:
 
-            logger.debug("running for %s" % str(result))
 
-            link = result.xpath('.//h2/a')[0]
 
-            url = link.attrib.get('href')
 
-            title = result.xpath('string(.//h2/a)')
 
+            url = result.xpath('.//h2/a')[0].get('href')
 
+            url = urljoin(base_url, url)
 
+            title = result.xpath('string(.//h2/a)').strip()
 
             content = extract_text(result.xpath('.//p'))
 
             content = extract_text(result.xpath('.//p'))
 
             # append result
 
             # append result
 
             results.append({'url': url,
 
             results.append({'url': url,

+ 17 - 11
tests/unit/engines/test_duden.py
View File 

@@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase):
 
     def test_request(self):
 
     def test_request(self):
 
         query = 'Haus'
 
         query = 'Haus'
 
         dic = defaultdict(dict)
 
         dic = defaultdict(dict)
 
-        dic['pageno'] = 1
 
-        params = duden.request(query, dic)
 
-        self.assertTrue('url' in params)
 
-        self.assertTrue(query in params['url'])
 
-        self.assertTrue('duden.de' in params['url'])
 
+        data = [
 
+            [1, 'https://www.duden.de/suchen/dudenonline/Haus'],
 
+            [2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1']
 
+        ]
 
+        for page_no, exp_res in data:
 
+            dic['pageno'] = page_no
 
+            params = duden.request(query, dic)
 
+            self.assertTrue('url' in params)
 
+            self.assertTrue(query in params['url'])
 
+            self.assertTrue('duden.de' in params['url'])
 
+            self.assertEqual(params['url'], exp_res)
 
 
 
     def test_response(self):
 
     def test_response(self):
 
         resp = mock.Mock(text='<html></html>')
 
         resp = mock.Mock(text='<html></html>')
 
         self.assertEqual(duden.response(resp), [])
 
         self.assertEqual(duden.response(resp), [])
 
 
 
         html = """
 
         html = """
 
-        <section class="wide">
 
-        <h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2>
 
-        <p>This is the <strong>content</strong></p>
 
-        <a href="https://this.is.the.url/">Zum vollst&auml;ndigen Artikel</a>
 
+        <section class="vignette">
 
+            <h2"> <a href="/rechtschreibung/Haus">
 
+                <strong>This is the title also here</strong>
 
+            </a> </h2>
 
+          <p>This is the content</p>
 
         </section>
 
         </section>
 
         """
 
         """
 
-
 
         resp = mock.Mock(text=html)
 
         resp = mock.Mock(text=html)
 
         results = duden.response(resp)
 
         results = duden.response(resp)
 
 
 
@@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase):
 
 
 
         # testing result (dictionary entry)
 
         # testing result (dictionary entry)
 
         r = results[0]
 
         r = results[0]
 
-        self.assertEqual(r['url'], 'https://this.is.the.url/')
 
+        self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus')
 
         self.assertEqual(r['title'], 'This is the title also here')
 
         self.assertEqual(r['title'], 'This is the title also here')
 
         self.assertEqual(r['content'], 'This is the content')
 
         self.assertEqual(r['content'], 'This is the content')

© 2025 NTab仓库
Page: 952ms Template: 311ms 微信 微信二维码 哔哩哔哩 Sina Weibo Javascript Licenses Website