webapp.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885
  1. #!/usr/bin/env python
  2. '''
  3. searx is free software: you can redistribute it and/or modify
  4. it under the terms of the GNU Affero General Public License as published by
  5. the Free Software Foundation, either version 3 of the License, or
  6. (at your option) any later version.
  7. searx is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Affero General Public License for more details.
  11. You should have received a copy of the GNU Affero General Public License
  12. along with searx. If not, see < http://www.gnu.org/licenses/ >.
  13. (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
  14. '''
  15. if __name__ == '__main__':
  16. from sys import path
  17. from os.path import realpath, dirname
  18. path.append(realpath(dirname(realpath(__file__)) + '/../'))
  19. import hashlib
  20. import hmac
  21. import json
  22. import os
  23. import sys
  24. import requests
  25. from searx import logger
  26. logger = logger.getChild('webapp')
  27. try:
  28. from pygments import highlight
  29. from pygments.lexers import get_lexer_by_name
  30. from pygments.formatters import HtmlFormatter
  31. except:
  32. logger.critical("cannot import dependency: pygments")
  33. from sys import exit
  34. exit(1)
  35. from cgi import escape
  36. from datetime import datetime, timedelta
  37. from werkzeug.contrib.fixers import ProxyFix
  38. from flask import (
  39. Flask, request, render_template, url_for, Response, make_response,
  40. redirect, send_from_directory
  41. )
  42. from flask_babel import Babel, gettext, format_date, format_decimal
  43. from flask.json import jsonify
  44. from searx import settings, searx_dir, searx_debug
  45. from searx.exceptions import SearxParameterException
  46. from searx.engines import (
  47. categories, engines, engine_shortcuts, get_engines_stats, initialize_engines
  48. )
  49. from searx.utils import (
  50. UnicodeWriter, highlight_content, html_to_text, get_resources_directory,
  51. get_static_files, get_result_templates, get_themes, gen_useragent,
  52. dict_subset, prettify_url
  53. )
  54. from searx.version import VERSION_STRING
  55. from searx.languages import language_codes
  56. from searx.search import SearchWithPlugins, get_search_query_from_webapp
  57. from searx.query import RawTextQuery
  58. from searx.autocomplete import searx_bang, backends as autocomplete_backends
  59. from searx.plugins import plugins
  60. from searx.preferences import Preferences, ValidationException
  61. from searx.answerers import answerers
  62. from searx.url_utils import urlencode, urlparse, urljoin
  63. # check if the pyopenssl package is installed.
  64. # It is needed for SSL connection without trouble, see #298
  65. try:
  66. import OpenSSL.SSL # NOQA
  67. except ImportError:
  68. logger.critical("The pyopenssl package has to be installed.\n"
  69. "Some HTTPS connections will fail")
  70. try:
  71. from cStringIO import StringIO
  72. except:
  73. from io import StringIO
  74. if sys.version_info[0] == 3:
  75. unicode = str
  76. # serve pages with HTTP/1.1
  77. from werkzeug.serving import WSGIRequestHandler
  78. WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
  79. # about static
  80. static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
  81. logger.debug('static directory is %s', static_path)
  82. static_files = get_static_files(static_path)
  83. # about templates
  84. default_theme = settings['ui']['default_theme']
  85. templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
  86. logger.debug('templates directory is %s', templates_path)
  87. themes = get_themes(templates_path)
  88. result_templates = get_result_templates(templates_path)
  89. global_favicons = []
  90. for indice, theme in enumerate(themes):
  91. global_favicons.append([])
  92. theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
  93. for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
  94. global_favicons[indice].extend(filenames)
  95. # Flask app
  96. app = Flask(
  97. __name__,
  98. static_folder=static_path,
  99. template_folder=templates_path
  100. )
  101. app.jinja_env.trim_blocks = True
  102. app.jinja_env.lstrip_blocks = True
  103. app.secret_key = settings['server']['secret_key']
  104. if not searx_debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true":
  105. initialize_engines(settings['engines'])
  106. babel = Babel(app)
  107. rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he',
  108. 'ku', 'mzn', 'pnb'', ''ps', 'sd', 'ug', 'ur', 'yi']
  109. # used when translating category names
  110. _category_names = (gettext('files'),
  111. gettext('general'),
  112. gettext('music'),
  113. gettext('social media'),
  114. gettext('images'),
  115. gettext('videos'),
  116. gettext('it'),
  117. gettext('news'),
  118. gettext('map'),
  119. gettext('science'))
  120. outgoing_proxies = settings['outgoing'].get('proxies', None)
  121. @babel.localeselector
  122. def get_locale():
  123. locale = request.accept_languages.best_match(settings['locales'].keys())
  124. if request.preferences.get_value('locale') != '':
  125. locale = request.preferences.get_value('locale')
  126. if 'locale' in request.args\
  127. and request.args['locale'] in settings['locales']:
  128. locale = request.args['locale']
  129. if 'locale' in request.form\
  130. and request.form['locale'] in settings['locales']:
  131. locale = request.form['locale']
  132. return locale
  133. # code-highlighter
  134. @app.template_filter('code_highlighter')
  135. def code_highlighter(codelines, language=None):
  136. if not language:
  137. language = 'text'
  138. try:
  139. # find lexer by programing language
  140. lexer = get_lexer_by_name(language, stripall=True)
  141. except:
  142. # if lexer is not found, using default one
  143. logger.debug('highlighter cannot find lexer for {0}'.format(language))
  144. lexer = get_lexer_by_name('text', stripall=True)
  145. html_code = ''
  146. tmp_code = ''
  147. last_line = None
  148. # parse lines
  149. for line, code in codelines:
  150. if not last_line:
  151. line_code_start = line
  152. # new codeblock is detected
  153. if last_line is not None and\
  154. last_line + 1 != line:
  155. # highlight last codepart
  156. formatter = HtmlFormatter(linenos='inline',
  157. linenostart=line_code_start)
  158. html_code = html_code + highlight(tmp_code, lexer, formatter)
  159. # reset conditions for next codepart
  160. tmp_code = ''
  161. line_code_start = line
  162. # add codepart
  163. tmp_code += code + '\n'
  164. # update line
  165. last_line = line
  166. # highlight last codepart
  167. formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start)
  168. html_code = html_code + highlight(tmp_code, lexer, formatter)
  169. return html_code
  170. # Extract domain from url
  171. @app.template_filter('extract_domain')
  172. def extract_domain(url):
  173. return urlparse(url)[1]
  174. def get_base_url():
  175. if settings['server']['base_url']:
  176. hostname = settings['server']['base_url']
  177. else:
  178. scheme = 'http'
  179. if request.is_secure:
  180. scheme = 'https'
  181. hostname = url_for('index', _external=True, _scheme=scheme)
  182. return hostname
  183. def get_current_theme_name(override=None):
  184. """Returns theme name.
  185. Checks in this order:
  186. 1. override
  187. 2. cookies
  188. 3. settings"""
  189. if override and (override in themes or override == '__common__'):
  190. return override
  191. theme_name = request.args.get('theme', request.preferences.get_value('theme'))
  192. if theme_name not in themes:
  193. theme_name = default_theme
  194. return theme_name
  195. def get_result_template(theme, template_name):
  196. themed_path = theme + '/result_templates/' + template_name
  197. if themed_path in result_templates:
  198. return themed_path
  199. return 'result_templates/' + template_name
  200. def url_for_theme(endpoint, override_theme=None, **values):
  201. if endpoint == 'static' and values.get('filename'):
  202. theme_name = get_current_theme_name(override=override_theme)
  203. filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
  204. if filename_with_theme in static_files:
  205. values['filename'] = filename_with_theme
  206. return url_for(endpoint, **values)
  207. def proxify(url):
  208. if url.startswith('//'):
  209. url = 'https:' + url
  210. if not settings.get('result_proxy'):
  211. return url
  212. url_params = dict(mortyurl=url.encode('utf-8'))
  213. if settings['result_proxy'].get('key'):
  214. url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'],
  215. url.encode('utf-8'),
  216. hashlib.sha256).hexdigest()
  217. return '{0}?{1}'.format(settings['result_proxy']['url'],
  218. urlencode(url_params))
  219. def image_proxify(url):
  220. if url.startswith('//'):
  221. url = 'https:' + url
  222. if not request.preferences.get_value('image_proxy'):
  223. return url
  224. if settings.get('result_proxy'):
  225. return proxify(url)
  226. h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
  227. return '{0}?{1}'.format(url_for('image_proxy'),
  228. urlencode(dict(url=url.encode('utf-8'), h=h)))
  229. def render(template_name, override_theme=None, **kwargs):
  230. disabled_engines = request.preferences.engines.get_disabled()
  231. enabled_categories = set(category for engine_name in engines
  232. for category in engines[engine_name].categories
  233. if (engine_name, category) not in disabled_engines)
  234. if 'categories' not in kwargs:
  235. kwargs['categories'] = ['general']
  236. kwargs['categories'].extend(x for x in
  237. sorted(categories.keys())
  238. if x != 'general'
  239. and x in enabled_categories)
  240. if 'all_categories' not in kwargs:
  241. kwargs['all_categories'] = ['general']
  242. kwargs['all_categories'].extend(x for x in
  243. sorted(categories.keys())
  244. if x != 'general')
  245. if 'selected_categories' not in kwargs:
  246. kwargs['selected_categories'] = []
  247. for arg in request.args:
  248. if arg.startswith('category_'):
  249. c = arg.split('_', 1)[1]
  250. if c in categories:
  251. kwargs['selected_categories'].append(c)
  252. if not kwargs['selected_categories']:
  253. cookie_categories = request.preferences.get_value('categories')
  254. for ccateg in cookie_categories:
  255. kwargs['selected_categories'].append(ccateg)
  256. if not kwargs['selected_categories']:
  257. kwargs['selected_categories'] = ['general']
  258. if 'autocomplete' not in kwargs:
  259. kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
  260. if get_locale() in rtl_locales and 'rtl' not in kwargs:
  261. kwargs['rtl'] = True
  262. kwargs['searx_version'] = VERSION_STRING
  263. kwargs['method'] = request.preferences.get_value('method')
  264. kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
  265. kwargs['language_codes'] = language_codes
  266. if 'current_language' not in kwargs:
  267. kwargs['current_language'] = request.preferences.get_value('language')
  268. # override url_for function in templates
  269. kwargs['url_for'] = url_for_theme
  270. kwargs['image_proxify'] = image_proxify
  271. kwargs['proxify'] = proxify if settings.get('result_proxy') else None
  272. kwargs['get_result_template'] = get_result_template
  273. kwargs['theme'] = get_current_theme_name(override=override_theme)
  274. kwargs['template_name'] = template_name
  275. kwargs['cookies'] = request.cookies
  276. kwargs['errors'] = request.errors
  277. kwargs['instance_name'] = settings['general']['instance_name']
  278. kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')
  279. kwargs['unicode'] = unicode
  280. kwargs['scripts'] = set()
  281. for plugin in request.user_plugins:
  282. for script in plugin.js_dependencies:
  283. kwargs['scripts'].add(script)
  284. kwargs['styles'] = set()
  285. for plugin in request.user_plugins:
  286. for css in plugin.css_dependencies:
  287. kwargs['styles'].add(css)
  288. return render_template(
  289. '{}/{}'.format(kwargs['theme'], template_name), **kwargs)
  290. @app.before_request
  291. def pre_request():
  292. request.errors = []
  293. preferences = Preferences(themes, list(categories.keys()), engines, plugins)
  294. request.preferences = preferences
  295. try:
  296. preferences.parse_cookies(request.cookies)
  297. except:
  298. request.errors.append(gettext('Invalid settings, please edit your preferences'))
  299. # merge GET, POST vars
  300. # request.form
  301. request.form = dict(request.form.items())
  302. for k, v in request.args.items():
  303. if k not in request.form:
  304. request.form[k] = v
  305. # request.user_plugins
  306. request.user_plugins = []
  307. allowed_plugins = preferences.plugins.get_enabled()
  308. disabled_plugins = preferences.plugins.get_disabled()
  309. for plugin in plugins:
  310. if ((plugin.default_on and plugin.id not in disabled_plugins)
  311. or plugin.id in allowed_plugins):
  312. request.user_plugins.append(plugin)
  313. def index_error(output_format, error_message):
  314. if output_format == 'json':
  315. return Response(json.dumps({'error': error_message}),
  316. mimetype='application/json')
  317. elif output_format == 'csv':
  318. response = Response('', mimetype='application/csv')
  319. cont_disp = 'attachment;Filename=searx.csv'
  320. response.headers.add('Content-Disposition', cont_disp)
  321. return response
  322. elif output_format == 'rss':
  323. response_rss = render(
  324. 'opensearch_response_rss.xml',
  325. results=[],
  326. q=request.form['q'] if 'q' in request.form else '',
  327. number_of_results=0,
  328. base_url=get_base_url(),
  329. error_message=error_message
  330. )
  331. return Response(response_rss, mimetype='text/xml')
  332. else:
  333. # html
  334. request.errors.append(gettext('search error'))
  335. return render(
  336. 'index.html',
  337. )
  338. @app.route('/search', methods=['GET', 'POST'])
  339. @app.route('/', methods=['GET', 'POST'])
  340. def index():
  341. """Render index page.
  342. Supported outputs: html, json, csv, rss.
  343. """
  344. # output_format
  345. output_format = request.form.get('format', 'html')
  346. if output_format not in ['html', 'csv', 'json', 'rss']:
  347. output_format = 'html'
  348. # check if there is query
  349. if request.form.get('q') is None:
  350. if output_format == 'html':
  351. return render(
  352. 'index.html',
  353. )
  354. else:
  355. return index_error(output_format, 'No query'), 400
  356. # search
  357. search_query = None
  358. result_container = None
  359. try:
  360. search_query = get_search_query_from_webapp(request.preferences, request.form)
  361. # search = Search(search_query) # without plugins
  362. search = SearchWithPlugins(search_query, request.user_plugins, request)
  363. result_container = search.search()
  364. except Exception as e:
  365. # log exception
  366. logger.exception('search error')
  367. # is it an invalid input parameter or something else ?
  368. if (issubclass(e.__class__, SearxParameterException)):
  369. return index_error(output_format, e.message), 400
  370. else:
  371. return index_error(output_format, gettext('search error')), 500
  372. # results
  373. results = result_container.get_ordered_results()
  374. number_of_results = result_container.results_number()
  375. if number_of_results < result_container.results_length():
  376. number_of_results = 0
  377. # UI
  378. advanced_search = request.form.get('advanced_search', None)
  379. # output
  380. for result in results:
  381. if output_format == 'html':
  382. if 'content' in result and result['content']:
  383. result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)
  384. result['title'] = highlight_content(escape(result['title'] or u''), search_query.query)
  385. else:
  386. if result.get('content'):
  387. result['content'] = html_to_text(result['content']).strip()
  388. # removing html content and whitespace duplications
  389. result['title'] = ' '.join(html_to_text(result['title']).strip().split())
  390. result['pretty_url'] = prettify_url(result['url'])
  391. # TODO, check if timezone is calculated right
  392. if 'publishedDate' in result:
  393. try: # test if publishedDate >= 1900 (datetime module bug)
  394. result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
  395. except ValueError:
  396. result['publishedDate'] = None
  397. else:
  398. if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
  399. timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
  400. minutes = int((timedifference.seconds / 60) % 60)
  401. hours = int(timedifference.seconds / 60 / 60)
  402. if hours == 0:
  403. result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes)
  404. else:
  405. result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa
  406. else:
  407. result['publishedDate'] = format_date(result['publishedDate'])
  408. if output_format == 'json':
  409. return Response(json.dumps({'query': search_query.query.decode('utf-8'),
  410. 'number_of_results': number_of_results,
  411. 'results': results,
  412. 'answers': list(result_container.answers),
  413. 'corrections': list(result_container.corrections),
  414. 'infoboxes': result_container.infoboxes,
  415. 'suggestions': list(result_container.suggestions)}),
  416. mimetype='application/json')
  417. elif output_format == 'csv':
  418. csv = UnicodeWriter(StringIO())
  419. keys = ('title', 'url', 'content', 'host', 'engine', 'score')
  420. csv.writerow(keys)
  421. for row in results:
  422. row['host'] = row['parsed_url'].netloc
  423. csv.writerow([row.get(key, '') for key in keys])
  424. csv.stream.seek(0)
  425. response = Response(csv.stream.read(), mimetype='application/csv')
  426. cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)
  427. response.headers.add('Content-Disposition', cont_disp)
  428. return response
  429. elif output_format == 'rss':
  430. response_rss = render(
  431. 'opensearch_response_rss.xml',
  432. results=results,
  433. q=request.form['q'],
  434. number_of_results=number_of_results,
  435. base_url=get_base_url(),
  436. override_theme='__common__',
  437. )
  438. return Response(response_rss, mimetype='text/xml')
  439. return render(
  440. 'results.html',
  441. results=results,
  442. q=request.form['q'],
  443. selected_categories=search_query.categories,
  444. pageno=search_query.pageno,
  445. time_range=search_query.time_range,
  446. number_of_results=format_decimal(number_of_results),
  447. advanced_search=advanced_search,
  448. suggestions=result_container.suggestions,
  449. answers=result_container.answers,
  450. corrections=result_container.corrections,
  451. infoboxes=result_container.infoboxes,
  452. paging=result_container.paging,
  453. current_language=search_query.lang,
  454. base_url=get_base_url(),
  455. theme=get_current_theme_name(),
  456. favicons=global_favicons[themes.index(get_current_theme_name())]
  457. )
  458. @app.route('/about', methods=['GET'])
  459. def about():
  460. """Render about page"""
  461. return render(
  462. 'about.html',
  463. )
  464. @app.route('/autocompleter', methods=['GET', 'POST'])
  465. def autocompleter():
  466. """Return autocompleter results"""
  467. # set blocked engines
  468. disabled_engines = request.preferences.engines.get_disabled()
  469. # parse query
  470. raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)
  471. raw_text_query.parse_query()
  472. # check if search query is set
  473. if not raw_text_query.getSearchQuery():
  474. return '', 400
  475. # run autocompleter
  476. completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
  477. # parse searx specific autocompleter results like !bang
  478. raw_results = searx_bang(raw_text_query)
  479. # normal autocompletion results only appear if max 3 inner results returned
  480. if len(raw_results) <= 3 and completer:
  481. # get language from cookie
  482. language = request.preferences.get_value('language')
  483. if not language or language == 'all':
  484. language = 'en'
  485. else:
  486. language = language.split('-')[0]
  487. # run autocompletion
  488. raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
  489. # parse results (write :language and !engine back to result string)
  490. results = []
  491. for result in raw_results:
  492. raw_text_query.changeSearchQuery(result)
  493. # add parsed result
  494. results.append(raw_text_query.getFullQuery())
  495. # return autocompleter results
  496. if request.form.get('format') == 'x-suggestions':
  497. return Response(json.dumps([raw_text_query.query, results]),
  498. mimetype='application/json')
  499. return Response(json.dumps(results),
  500. mimetype='application/json')
  501. @app.route('/preferences', methods=['GET', 'POST'])
  502. def preferences():
  503. """Render preferences page && save user preferences"""
  504. # save preferences
  505. if request.method == 'POST':
  506. resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
  507. try:
  508. request.preferences.parse_form(request.form)
  509. except ValidationException:
  510. request.errors.append(gettext('Invalid settings, please edit your preferences'))
  511. return resp
  512. return request.preferences.save(resp)
  513. # render preferences
  514. image_proxy = request.preferences.get_value('image_proxy')
  515. lang = request.preferences.get_value('language')
  516. disabled_engines = request.preferences.engines.get_disabled()
  517. allowed_plugins = request.preferences.plugins.get_enabled()
  518. # stats for preferences page
  519. stats = {}
  520. for c in categories:
  521. for e in categories[c]:
  522. stats[e.name] = {'time': None,
  523. 'warn_timeout': False,
  524. 'warn_time': False}
  525. if e.timeout > settings['outgoing']['request_timeout']:
  526. stats[e.name]['warn_timeout'] = True
  527. # get first element [0], the engine time,
  528. # and then the second element [1] : the time (the first one is the label)
  529. for engine_stat in get_engines_stats()[0][1]:
  530. stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
  531. if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
  532. stats[engine_stat.get('name')]['warn_time'] = True
  533. # end of stats
  534. return render('preferences.html',
  535. locales=settings['locales'],
  536. current_locale=get_locale(),
  537. image_proxy=image_proxy,
  538. engines_by_category=categories,
  539. stats=stats,
  540. answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
  541. disabled_engines=disabled_engines,
  542. autocomplete_backends=autocomplete_backends,
  543. shortcuts={y: x for x, y in engine_shortcuts.items()},
  544. themes=themes,
  545. plugins=plugins,
  546. allowed_plugins=allowed_plugins,
  547. theme=get_current_theme_name(),
  548. preferences=True)
  549. @app.route('/image_proxy', methods=['GET'])
  550. def image_proxy():
  551. url = request.args.get('url').encode('utf-8')
  552. if not url:
  553. return '', 400
  554. h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest()
  555. if h != request.args.get('h'):
  556. return '', 400
  557. headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
  558. headers['User-Agent'] = gen_useragent()
  559. resp = requests.get(url,
  560. stream=True,
  561. timeout=settings['outgoing']['request_timeout'],
  562. headers=headers,
  563. proxies=outgoing_proxies)
  564. if resp.status_code == 304:
  565. return '', resp.status_code
  566. if resp.status_code != 200:
  567. logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
  568. if resp.status_code >= 400:
  569. return '', resp.status_code
  570. return '', 400
  571. if not resp.headers.get('content-type', '').startswith('image/'):
  572. logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
  573. return '', 400
  574. img = ''
  575. chunk_counter = 0
  576. for chunk in resp.iter_content(1024 * 1024):
  577. chunk_counter += 1
  578. if chunk_counter > 5:
  579. return '', 502 # Bad gateway - file is too big (>5M)
  580. img += chunk
  581. headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})
  582. return Response(img, mimetype=resp.headers['content-type'], headers=headers)
  583. @app.route('/stats', methods=['GET'])
  584. def stats():
  585. """Render engine statistics page."""
  586. stats = get_engines_stats()
  587. return render(
  588. 'stats.html',
  589. stats=stats,
  590. )
  591. @app.route('/robots.txt', methods=['GET'])
  592. def robots():
  593. return Response("""User-agent: *
  594. Allow: /
  595. Allow: /about
  596. Disallow: /stats
  597. Disallow: /preferences
  598. Disallow: /*?*q=*
  599. """, mimetype='text/plain')
  600. @app.route('/opensearch.xml', methods=['GET'])
  601. def opensearch():
  602. method = 'post'
  603. if request.preferences.get_value('method') == 'GET':
  604. method = 'get'
  605. # chrome/chromium only supports HTTP GET....
  606. if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
  607. method = 'get'
  608. ret = render('opensearch.xml',
  609. opensearch_method=method,
  610. host=get_base_url(),
  611. urljoin=urljoin,
  612. override_theme='__common__')
  613. resp = Response(response=ret,
  614. status=200,
  615. mimetype="text/xml")
  616. return resp
  617. @app.route('/favicon.ico')
  618. def favicon():
  619. return send_from_directory(os.path.join(app.root_path,
  620. 'static/themes',
  621. get_current_theme_name(),
  622. 'img'),
  623. 'favicon.png',
  624. mimetype='image/vnd.microsoft.icon')
  625. @app.route('/clear_cookies')
  626. def clear_cookies():
  627. resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
  628. for cookie_name in request.cookies:
  629. resp.delete_cookie(cookie_name)
  630. return resp
  631. @app.route('/config')
  632. def config():
  633. return jsonify({'categories': categories.keys(),
  634. 'engines': [{'name': engine_name,
  635. 'categories': engine.categories,
  636. 'shortcut': engine.shortcut,
  637. 'enabled': not engine.disabled,
  638. 'paging': engine.paging,
  639. 'language_support': engine.language_support,
  640. 'supported_languages':
  641. engine.supported_languages.keys()
  642. if isinstance(engine.supported_languages, dict)
  643. else engine.supported_languages,
  644. 'safesearch': engine.safesearch,
  645. 'time_range_support': engine.time_range_support,
  646. 'timeout': engine.timeout}
  647. for engine_name, engine in engines.items()],
  648. 'plugins': [{'name': plugin.name,
  649. 'enabled': plugin.default_on}
  650. for plugin in plugins],
  651. 'instance_name': settings['general']['instance_name'],
  652. 'locales': settings['locales'],
  653. 'default_locale': settings['ui']['default_locale'],
  654. 'autocomplete': settings['search']['autocomplete'],
  655. 'safe_search': settings['search']['safe_search'],
  656. 'default_theme': settings['ui']['default_theme'],
  657. 'version': VERSION_STRING})
  658. @app.errorhandler(404)
  659. def page_not_found(e):
  660. return render('404.html'), 404
  661. def run():
  662. logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address'])
  663. app.run(
  664. debug=searx_debug,
  665. use_debugger=searx_debug,
  666. port=settings['server']['port'],
  667. host=settings['server']['bind_address'],
  668. threaded=True
  669. )
  670. class ReverseProxyPathFix(object):
  671. '''Wrap the application in this middleware and configure the
  672. front-end server to add these headers, to let you quietly bind
  673. this to a URL other than / and to an HTTP scheme that is
  674. different than what is used locally.
  675. http://flask.pocoo.org/snippets/35/
  676. In nginx:
  677. location /myprefix {
  678. proxy_pass http://127.0.0.1:8000;
  679. proxy_set_header Host $host;
  680. proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  681. proxy_set_header X-Scheme $scheme;
  682. proxy_set_header X-Script-Name /myprefix;
  683. }
  684. :param app: the WSGI application
  685. '''
  686. def __init__(self, app):
  687. self.app = app
  688. def __call__(self, environ, start_response):
  689. script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
  690. if script_name:
  691. environ['SCRIPT_NAME'] = script_name
  692. path_info = environ['PATH_INFO']
  693. if path_info.startswith(script_name):
  694. environ['PATH_INFO'] = path_info[len(script_name):]
  695. scheme = environ.get('HTTP_X_SCHEME', '')
  696. if scheme:
  697. environ['wsgi.url_scheme'] = scheme
  698. return self.app(environ, start_response)
  699. application = app
  700. # patch app to handle non root url-s behind proxy & wsgi
  701. app.wsgi_app = ReverseProxyPathFix(ProxyFix(application.wsgi_app))
  702. if __name__ == "__main__":
  703. run()