NTab仓库
fjmwkj
/
searxng
mirror of https://github.com/searxng/searxng
1
0
Fork 0
Files Issues 0 Wiki
Browse Source

Fix bing engine results count (#1387)

This PR fixes the result count from bing which was throwing an (hidden) error and add a validation to avoid reading more results than avalaible.

For example :
If there is 100 results from some search and we try to get results from 120 to 130, Bing will send back the results from 0 to 10 and no error. If we compare results count with the first parameter of the request we can avoid this "invalid" results.
Léo Bourrel 5 years ago
parent
12f891da84
commit
88261e111c
2 changed files with 119 additions and 43 deletions
Split View Show Diff Stats
  1. 28 9
      searx/engines/bing.py
  2. 91 34
      tests/unit/engines/test_bing.py

+ 28 - 9
searx/engines/bing.py
View File 

@@ -13,11 +13,15 @@
 
  @todo        publishedDate
 
 """
 
 
+import re
 
 from lxml import html
 
+from searx import logger, utils
 
 from searx.engines.xpath import extract_text
 
 from searx.url_utils import urlencode
 
 from searx.utils import match_language, gen_useragent
 
 
+logger = logger.getChild('bing engine')
 
+
 
 # engine dependent config
 
 categories = ['general']
 
 paging = True
 
@@ -30,9 +34,13 @@ base_url = 'https://www.bing.com/'
 
 search_string = 'search?{query}&first={offset}'
 
 
 
+def _get_offset_from_pageno(pageno):
 
+    return (pageno - 1) * 10 + 1
 
+
 
+
 
 # do search-request
 
 def request(query, params):
 
-    offset = (params['pageno'] - 1) * 10 + 1
 
+    offset = _get_offset_from_pageno(params.get('pageno', 0))
 
 
     if params['language'] == 'all':
 
         lang = 'EN'
 
@@ -53,15 +61,9 @@ def request(query, params):
 
 # get response from search-request
 
 def response(resp):
 
     results = []
 
+    result_len = 0
 
 
     dom = html.fromstring(resp.text)
 
-
 
-    try:
 
-        results.append({'number_of_results': int(dom.xpath('//span[@class="sb_count"]/text()')[0]
 
-                                                 .split()[0].replace(',', ''))})
 
-    except:
 
-        pass
 
-
 
     # parse results
 
     for result in dom.xpath('//div[@class="sa_cc"]'):
 
         link = result.xpath('.//h3/a')[0]
 
@@ -86,7 +88,24 @@ def response(resp):
 
                         'title': title,
 
                         'content': content})
 
 
-    # return results
 
+    try:
 
+        result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))
 
+        result_len_container = utils.to_string(result_len_container)
 
+        if "-" in result_len_container:
 
+            # Remove the part "from-to" for paginated request ...
 
+            result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
 
+
 
+        result_len_container = re.sub('[^0-9]', '', result_len_container)
 
+        if len(result_len_container) > 0:
 
+            result_len = int(result_len_container)
 
+    except Exception as e:
 
+        logger.debug('result error :\n%s', e)
 
+        pass
 
+
 
+    if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
 
+        return []
 
+
 
+    results.append({'number_of_results': result_len})
 
     return results

+ 91 - 34
tests/unit/engines/test_bing.py
View File 

@@ -1,3 +1,4 @@
 
+# -*- coding: utf-8 -*-
 
 from collections import defaultdict
 
 import mock
 
 from searx.engines import bing
 
@@ -10,7 +11,7 @@ class TestBingEngine(SearxTestCase):
 
         bing.supported_languages = ['en', 'fr', 'zh-CHS', 'zh-CHT', 'pt-PT', 'pt-BR']
 
         query = u'test_query'
 
         dicto = defaultdict(dict)
 
-        dicto['pageno'] = 0
 
+        dicto['pageno'] = 1
 
         dicto['language'] = 'fr-FR'
 
         params = bing.request(query.encode('utf-8'), dicto)
 
         self.assertTrue('url' in params)
 
@@ -23,70 +24,126 @@ class TestBingEngine(SearxTestCase):
 
         self.assertTrue('language' in params['url'])
 
 
     def test_response(self):
 
+        dicto = defaultdict(dict)
 
+        dicto['pageno'] = 1
 
+        dicto['language'] = 'fr-FR'
 
         self.assertRaises(AttributeError, bing.response, None)
 
         self.assertRaises(AttributeError, bing.response, [])
 
         self.assertRaises(AttributeError, bing.response, '')
 
         self.assertRaises(AttributeError, bing.response, '[]')
 
 
         response = mock.Mock(text='<html></html>')
 
+        response.search_params = dicto
 
         self.assertEqual(bing.response(response), [])
 
 
         response = mock.Mock(text='<html></html>')
 
+        response.search_params = dicto
 
         self.assertEqual(bing.response(response), [])
 
 
         html = """
 
-        <div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
 
-            <div Class="sa_mc">
 
-                <div class="sb_tlst">
 
-                    <h3>
 
-                        <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
 
-                        <strong>This</strong> should be the title</a>
 
-                    </h3>
 
-                </div>
 
-                <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
 
-                    <span class="c_tlbxTrg">
 
-                        <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
 
-                        </span>
 
-                    </span>
 
-                </div>
 
-                <p><strong>This</strong> should be the content.</p>
 
+        <div>
 
+            <div id="b_tween">
 
+                <span class="sb_count" data-bm="4">23 900 000 résultats</span>
 
             </div>
 
+            <ol id="b_results" role="main">
 
+                <div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
 
+                    <div Class="sa_mc">
 
+                        <div class="sb_tlst">
 
+                            <h3>
 
+                                <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
 
+                                <strong>This</strong> should be the title</a>
 
+                            </h3>
 
+                        </div>
 
+                        <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
 
+                            <span class="c_tlbxTrg">
 
+                                <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
 
+                                </span>
 
+                            </span>
 
+                        </div>
 
+                        <p><strong>This</strong> should be the content.</p>
 
+                    </div>
 
+                </div>
 
+            </ol>
 
         </div>
 
         """
 
         response = mock.Mock(text=html)
 
+        response.search_params = dicto
 
         results = bing.response(response)
 
         self.assertEqual(type(results), list)
 
-        self.assertEqual(len(results), 1)
 
+        self.assertEqual(len(results), 2)
 
         self.assertEqual(results[0]['title'], 'This should be the title')
 
         self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
 
         self.assertEqual(results[0]['content'], 'This should be the content.')
 
+        self.assertEqual(results[-1]['number_of_results'], 23900000)
 
 
         html = """
 
-        <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
 
-            <div Class="sa_mc">
 
-                <div class="sb_tlst">
 
-                    <h2>
 
-                        <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
 
-                        <strong>This</strong> should be the title</a>
 
-                    </h2>
 
-                </div>
 
-                <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
 
-                    <span class="c_tlbxTrg">
 
-                        <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
 
-                        </span>
 
-                    </span>
 
-                </div>
 
-                <p><strong>This</strong> should be the content.</p>
 
+        <div>
 
+            <div id="b_tween">
 
+                <span class="sb_count" data-bm="4">9-18 résultats sur 23 900 000</span>
 
             </div>
 
-        </li>
 
+            <ol id="b_results" role="main">
 
+                <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
 
+                    <div Class="sa_mc">
 
+                        <div class="sb_tlst">
 
+                            <h2>
 
+                                <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
 
+                                <strong>This</strong> should be the title</a>
 
+                            </h2>
 
+                        </div>
 
+                        <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
 
+                            <span class="c_tlbxTrg">
 
+                                <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
 
+                                </span>
 
+                            </span>
 
+                        </div>
 
+                        <p><strong>This</strong> should be the content.</p>
 
+                    </div>
 
+                </li>
 
+            </ol>
 
+        </div>
 
         """
 
+        dicto['pageno'] = 2
 
         response = mock.Mock(text=html)
 
+        response.search_params = dicto
 
         results = bing.response(response)
 
         self.assertEqual(type(results), list)
 
-        self.assertEqual(len(results), 1)
 
+        self.assertEqual(len(results), 2)
 
         self.assertEqual(results[0]['title'], 'This should be the title')
 
         self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
 
         self.assertEqual(results[0]['content'], 'This should be the content.')
 
+        self.assertEqual(results[-1]['number_of_results'], 23900000)
 
+
 
+        html = """
 
+        <div>
 
+            <div id="b_tween">
 
+                <span class="sb_count" data-bm="4">23 900 000 résultats</span>
 
+            </div>
 
+            <ol id="b_results" role="main">
 
+                <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
 
+                    <div Class="sa_mc">
 
+                        <div class="sb_tlst">
 
+                            <h2>
 
+                                <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
 
+                                <strong>This</strong> should be the title</a>
 
+                            </h2>
 
+                        </div>
 
+                        <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite>
 
+                            <span class="c_tlbxTrg">
 
+                                <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
 
+                                </span>
 
+                            </span>
 
+                        </div>
 
+                        <p><strong>This</strong> should be the content.</p>
 
+                    </div>
 
+                </li>
 
+            </ol>
 
+        </div>
 
+        """
 
+        dicto['pageno'] = 33900000
 
+        response = mock.Mock(text=html)
 
+        response.search_params = dicto
 
+        results = bing.response(response)
 
+        self.assertEqual(bing.response(response), [])
 
 
     def test_fetch_supported_languages(self):
 
         html = """<html></html>"""

© 2025 NTab仓库
Page: 55ms Template: 27ms 微信 微信二维码 哔哩哔哩 Sina Weibo Javascript Licenses Website