Browse Source

Merge branch 'master' into bug/oscar-theme

Markus Heiser 5 years ago
parent
commit
1d86d0054f

+ 4 - 0
Makefile

@@ -8,6 +8,9 @@ PYOBJECTS = searx
 DOC       = docs
 PY_SETUP_EXTRAS ?= \[test\]
 
+PYDIST=./dist/py
+PYBUILD=./build/py
+
 include utils/makefile.include
 include utils/makefile.python
 include utils/makefile.sphinx
@@ -23,6 +26,7 @@ help:
 	@echo  '  install   - developer install (./local)'
 	@echo  '  uninstall - uninstall (./local)'
 	@echo  '  gh-pages  - build docs & deploy on gh-pages branch'
+	@echo  '  clean     - drop builds and environments'
 	@echo  ''
 	@$(MAKE) -s -f utils/makefile.include make-help
 	@echo  ''

+ 3 - 3
README.rst

@@ -23,13 +23,13 @@ Go to the `searx-docker <https://github.com/searx/searx-docker>`__ project.
 
 Without Docker
 ------
-For all the details, follow this `step by step installation <https://asciimoo.github.io/searx/dev/install/installation.html>`__.
+For all of the details, follow this `step by step installation <https://asciimoo.github.io/searx/dev/install/installation.html>`__.
 
 Note: the documentation needs to be updated.
 
-If you are in hurry
+If you are in a hurry
 ------
--  clone source:
+-  clone the source:
    ``git clone https://github.com/asciimoo/searx.git && cd searx``
 -  install dependencies: ``./manage.sh update_packages``
 -  edit your

+ 101 - 1
docs/_themes/searx/static/searx.css

@@ -24,7 +24,107 @@ p.sidebar-title, .sidebar p {
   margin: 6pt;
 }
 
-.sidebar li {
+.sidebar li,
+.hlist li {
   list-style-type: disclosure-closed;
 }
 
+
+/* admonitions
+*/
+
+div.admonition, div.topic {
+  background-color: #fafafa;
+  margin: 8px 0px;
+  padding: 1em;
+  border-radius: 3pt 0 0 3pt;
+  border-top: none;
+  border-right: none;
+  border-bottom: none;
+  border-left: 5pt solid #ccc;
+}
+
+p.admonition-title:after {
+  content: none;
+}
+
+.admonition.hint      { border-color: #416dc0b0; }
+.admonition.note      { border-color: #6c856cb0; }
+.admonition.tip       { border-color: #85c5c2b0; }
+.admonition.attention { border-color: #ecec97b0; }
+.admonition.caution   { border-color: #a6c677b0; }
+.admonition.danger    { border-color: #d46262b0; }
+.admonition.important { border-color: #dfa3a3b0; }
+.admonition.error     { border-color: red; }
+.admonition.warning   { border-color: darkred; }
+
+.admonition.admonition-generic-admonition-title {
+    border-color: #416dc0b0;
+}
+
+
+/* admonitions with (rendered) reST markup examples (:class: rst-example)
+ *
+ * .. admonition:: title of the example
+ *     :class: rst-example
+ *     ....
+*/
+
+div.rst-example {
+  background-color: inherit;
+  margin: 0;
+  border-top: none;
+  border-right: 1px solid #ccc;
+  border-bottom: none;
+  border-left: none;
+  border-radius: none;
+  padding: 0;
+}
+
+div.rst-example > p.admonition-title {
+  font-family: Sans Serif;
+  font-style: italic;
+  font-size: 0.8em;
+  display: block;
+  border-bottom: 1px solid #ccc;
+  padding: 0.5em 1em;
+  text-align: right;
+}
+
+/* code block in figures
+ */
+
+div.highlight pre {
+    text-align: left;
+}
+
+/* Table theme
+*/
+
+thead, tfoot {
+  background-color: #fff;
+}
+
+th:hover, td:hover {
+  background-color: #ffc;
+}
+
+thead th, tfoot th, tfoot td, tbody th {
+  background-color: #fffaef;
+}
+
+tbody tr:nth-child(odd) {
+  background-color: #fff;
+}
+
+tbody tr:nth-child(even) {
+  background-color: #fafafa;
+}
+
+caption {
+  font-family: Sans Serif;
+  padding: 0.5em;
+  margin: 0.5em 0 0.5em 0;
+  caption-side: top;
+  text-align: left;
+}

+ 33 - 0
docs/admin/arch_public.dot

@@ -0,0 +1,33 @@
+digraph G {
+
+  node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"];
+  edge [fontname="Sans"];
+
+  browser [label="Browser", shape=Mdiamond];
+  rp      [label="Reverse Proxy", href="url to configure reverse proxy"];
+  filtron [label="Filtron",       href="https://github.com/asciimoo/filtron"];
+  morty   [label="Morty",         href="https://github.com/asciimoo/morty"];
+  static  [label="Static files",  href="url to configure static files"];
+  uwsgi   [label="uwsgi",         href="url to configure uwsgi"]
+  searx1  [label="Searx #1"];
+  searx2  [label="Searx #2"];
+  searx3  [label="Searx #3"];
+  searx4  [label="Searx #4"];
+
+  browser -> rp [label="HTTPS"]
+
+  subgraph cluster_searx {
+      label = "Searx instance" fontname="Sans";
+      bgcolor="#fafafa";
+      { rank=same; static rp };
+      rp -> morty      [label="optional: images and HTML pages proxy"];
+      rp -> static     [label="optional: reverse proxy serves directly static files"];
+      rp -> filtron    [label="HTTP"];
+      filtron -> uwsgi [label="HTTP"];
+      uwsgi -> searx1;
+      uwsgi -> searx2;
+      uwsgi -> searx3;
+      uwsgi -> searx4;
+  }
+
+}

+ 24 - 0
docs/admin/architecture.rst

@@ -0,0 +1,24 @@
+.. _architecture:
+
+============
+Architecture
+============
+
+.. sidebar:: Needs work!
+
+   This article needs some work / Searx is a collaborative effort.  If you have
+   any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see
+   :ref:`how to contribute`.
+
+Herein you will find some hints and suggestions about typical architectures of
+searx infrastructures.
+
+We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
+It shows a *reference* setup for public searx instances.
+
+.. _arch public:
+
+.. kernel-figure:: arch_public.dot
+   :alt: arch_public.dot
+
+   Reference architecture of a public searx setup.

+ 103 - 0
docs/admin/buildhosts.rst

@@ -0,0 +1,103 @@
+.. _buildhosts:
+
+==========
+Buildhosts
+==========
+
+.. sidebar:: This article needs some work
+
+   If you have any contribution send us your :pull:`PR <../pulls>`, see
+   :ref:`how to contribute`.
+
+To get best results from build, its recommend to install additional packages
+on build hosts.
+
+.. _docs build:
+
+Build docs
+==========
+
+.. _Graphviz: https://graphviz.gitlab.io
+.. _ImageMagick: https://www.imagemagick.org
+.. _XeTeX: https://tug.org/xetex/
+.. _dvisvgm: https://dvisvgm.de/
+
+.. sidebar:: Sphinx build needs
+
+   - ImageMagick_
+   - Graphviz_
+   - XeTeX_
+   - dvisvgm_
+
+Most of the sphinx requirements are installed from :origin:`setup.py` and the
+docs can be build from scratch with ``make docs``.  For better math and image
+processing additional packages are needed.  The XeTeX_ needed not only for PDF
+creation, its also needed for :ref:`math` when HTML output is build.
+
+To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
+as images (``sphinx.ext.imgmath`` extension).  If your docs build (``make
+docs``) shows warnings like this::
+
+   WARNING: dot(1) not found, for better output quality install \
+            graphviz from http://www.graphviz.org
+   ..
+   WARNING: LaTeX command 'latex' cannot be run (needed for math \
+            display), check the imgmath_latex setting
+
+you need to install additional packages on your build host, to get better HTML
+output.
+
+.. _system requirements:
+
+.. tabs::
+
+   .. group-tab:: Ubuntu / debian
+
+      .. code-block:: sh
+
+         $ sudo apt install graphviz imagemagick texlive-xetex librsvg2-bin
+
+   .. group-tab:: Arch Linux
+
+      .. code-block:: sh
+
+         $ sudo pacman -S graphviz imagemagick texlive-bin extra/librsvg
+
+   .. group-tab::  Fedora / RHEL
+
+      .. code-block:: sh
+
+         $ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools
+
+
+For PDF output you also need:
+
+.. tabs::
+
+   .. group-tab:: Ubuntu / debian
+
+      .. code:: sh
+
+         $ sudo apt texlive-latex-recommended texlive-extra-utils ttf-dejavu
+
+   .. group-tab:: Arch Linux
+
+      .. code:: sh
+
+      	 $ sudo pacman -S texlive-core texlive-latexextra ttf-dejavu
+
+   .. group-tab::  Fedora / RHEL
+
+      .. code:: sh
+
+      	 $ sudo dnf install \
+	        texlive-collection-fontsrecommended texlive-collection-latex \
+		dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
+
+.. _system requirements END:
+
+.. literalinclude:: ../conf.py
+   :language: python
+   :start-after: # sphinx.ext.imgmath setup
+   :end-before: # sphinx.ext.imgmath setup END
+

+ 68 - 0
docs/admin/engines.rst

@@ -0,0 +1,68 @@
+.. _engines generic:
+
+=======
+engines
+=======
+
+.. sidebar:: Further reading ..
+
+   - :ref:`engines generic`
+   - :ref:`configured engines`
+   - :ref:`engine settings`
+   - :ref:`engine file`
+
+============= =========== ==================== ============
+:ref:`engine settings`    :ref:`engine file`
+------------------------- ---------------------------------
+Name (cfg)                Categories
+------------------------- ---------------------------------
+Engine        ..          Paging support       **P**
+------------------------- -------------------- ------------
+Shortcut      **S**       Language support     **L**
+Timeout       **TO**      Time range support   **TR**
+Disabled      **D**       Offline              **O**
+------------- ----------- -------------------- ------------
+Suspend end   **SE**
+------------- ----------- ---------------------------------
+Safe search   **SS**
+============= =========== =================================
+
+Configuration defaults (at built time):
+
+.. _configured engines:
+
+.. jinja:: webapp
+
+   .. flat-table:: Engines configured at built time (defaults)
+      :header-rows: 1
+      :stub-columns: 2
+
+      * - Name (cfg)
+        - S
+        - Engine
+        - TO
+        - Categories
+        - P
+        - L
+        - SS
+        - D
+        - TR
+        - O
+        - SE
+
+      {% for name, mod in engines.items() %}
+
+      * - {{name}}
+        - !{{mod.shortcut}}
+        - {{mod.__name__}}
+        - {{mod.timeout}}
+        - {{", ".join(mod.categories)}}
+        - {{(mod.paging and "y") or ""}}
+        - {{(mod.language_support and "y") or ""}}
+        - {{(mod.safesearch and "y") or ""}}
+        - {{(mod.disabled and "y") or ""}}
+        - {{(mod.time_range_support and "y") or ""}}
+        - {{(mod.offline and "y") or ""}}
+        - {{mod.suspend_end_time}}
+
+     {% endfor %}

+ 4 - 0
docs/admin/index.rst

@@ -7,5 +7,9 @@ Administrator documentation
 
    installation
    api
+   architecture
    filtron
    morty
+   engines
+   plugins
+   buildhosts

+ 15 - 7
docs/admin/installation.rst

@@ -114,6 +114,9 @@ content:
     # Module to import
     module = searx.webapp
 
+    # Support running the module from a webserver subdirectory.
+    route-run = fixpathinfo:
+
     # Virtualenv and python path
     virtualenv = /usr/local/searx/searx-ve/
     pythonpath = /usr/local/searx/
@@ -151,7 +154,10 @@ content:
     server {
         listen 80;
         server_name searx.example.com;
-        root /usr/local/searx;
+        root /usr/local/searx/searx;
+
+        location /static {
+        }
 
         location / {
                 include uwsgi_params;
@@ -180,14 +186,13 @@ Add this configuration in the server config file
 
 .. code:: nginx
 
-    location = /searx { rewrite ^ /searx/; }
-    location /searx {
-            try_files $uri @searx;
+    location /searx/static {
+            alias /usr/local/searx/searx/static;
     }
-    location @searx {
+
+    location /searx {
             uwsgi_param SCRIPT_NAME /searx;
             include uwsgi_params;
-            uwsgi_modifier1 30;
             uwsgi_pass unix:/run/uwsgi/app/searx/socket;
     }
 
@@ -197,6 +202,10 @@ in case of single-user or low-traffic instances.)
 
 .. code:: nginx
 
+    location /searx/static {
+            alias /usr/local/searx/searx/static;
+    }
+
     location /searx {
         proxy_pass http://127.0.0.1:8888;
         proxy_set_header Host $host;
@@ -338,4 +347,3 @@ References
 
 * How to: `Setup searx in a couple of hours with a free SSL certificate
   <https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__
-

+ 39 - 0
docs/admin/plugins.rst

@@ -0,0 +1,39 @@
+.. _plugins generic:
+
+===============
+Plugins builtin
+===============
+
+.. sidebar:: Further reading ..
+
+   - :ref:`dev plugin`
+
+Configuration defaults (at built time):
+
+:DO: Default on
+
+.. _configured plugins:
+
+.. jinja:: webapp
+
+   .. flat-table:: Plugins configured at built time (defaults)
+      :header-rows: 1
+      :stub-columns: 1
+      :widths: 3 1 9
+
+      * - Name
+        - DO
+        - Description
+
+          JS & CSS dependencies
+
+      {% for plgin in plugins %}
+
+      * - {{plgin.name}}
+        - {{(plgin.default_on and "y") or ""}}
+        - {{plgin.description}}
+
+          {% for dep in (plgin.js_dependencies + plgin.css_dependencies) %}
+          | ``{{dep}}`` {% endfor %}
+
+      {% endfor %}

+ 34 - 3
docs/conf.py

@@ -14,11 +14,18 @@ project = u'searx'
 copyright = u'2015-2019, Adam Tauber, Noémi Ványi'
 author = u'Adam Tauber'
 release, version = VERSION_STRING, VERSION_STRING
+highlight_language = 'none'
 
 # General --------------------------------------------------------------
 
 master_doc = "index"
 source_suffix = '.rst'
+numfig = True
+
+from searx import webapp
+jinja_contexts = {
+    'webapp': dict(**webapp.__dict__)
+}
 
 # usage::   lorem :patch:`f373169` ipsum
 extlinks = {}
@@ -32,21 +39,40 @@ extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://')
 extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
 extlinks['search'] = (SEARX_URL + '/%s', '#')
 extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
+extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ')
+extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
+#extlinks['role'] = (
+#    'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '')
+extlinks['duref'] = (
+    'http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '')
+extlinks['durole'] = (
+    'http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '')
+extlinks['dudir'] =  (
+    'http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '')
+extlinks['ctan'] =  (
+    'https://ctan.org/pkg/%s', 'CTAN: ')
 
 extensions = [
+    'sphinx.ext.imgmath',
     'sphinx.ext.extlinks',
     'sphinx.ext.viewcode',
     "sphinx.ext.autodoc",
     "sphinx.ext.intersphinx",
     "pallets_sphinx_themes",
     "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
+    "sphinxcontrib.jinja",  # https://github.com/tardyp/sphinx-jinja
+    'linuxdoc.rstFlatTable',    # Implementation of the 'flat-table' reST-directive.
+    'linuxdoc.kfigure',         # Sphinx extension which implements scalable image handling.
+    "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
 ]
 
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3/", None),
-    # "flask": ("https://flask.palletsprojects.com/", None),
+    "flask": ("https://flask.palletsprojects.com/", None),
     # "werkzeug": ("https://werkzeug.palletsprojects.com/", None),
-    # "jinja": ("https://jinja.palletsprojects.com/", None),
+    "jinja": ("https://jinja.palletsprojects.com/", None),
+    "linuxdoc" : ("https://return42.github.io/linuxdoc/", None),
+    "sphinx" : ("https://www.sphinx-doc.org/en/master/", None),
 }
 
 issues_github_path = "asciimoo/searx"
@@ -54,10 +80,15 @@ issues_github_path = "asciimoo/searx"
 # HTML -----------------------------------------------------------------
 
 sys.path.append(os.path.abspath('_themes'))
-
 html_theme_path = ['_themes']
 html_theme = "searx"
 
+# sphinx.ext.imgmath setup
+html_math_renderer = 'imgmath'
+imgmath_image_format = 'svg'
+imgmath_font_size = 14
+# sphinx.ext.imgmath setup END
+
 html_theme_options = {"index_sidebar_logo": True}
 html_context = {
     "project_links": [

+ 38 - 5
docs/dev/contribution_guide.rst

@@ -1,3 +1,5 @@
+.. _how to contribute:
+
 =================
 How to contribute
 =================
@@ -48,7 +50,20 @@ Code
 ====
 
 .. _PEP8: https://www.python.org/dev/peps/pep-0008/
+.. _Conventional Commits: https://www.conventionalcommits.org/
+.. _Git Commit Good Practice: https://wiki.openstack.org/wiki/GitCommitMessages
+.. _Structural split of changes:
+    https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes
+.. _gitmoji: https://gitmoji.carloscuesta.me/
+.. _Semantic PR: https://github.com/zeke/semantic-pull-requests
+
+.. sidebar:: Create good commits!
 
+   - `Structural split of changes`_
+   - `Conventional Commits`_
+   - `Git Commit Good Practice`_
+   - some like to use: gitmoji_
+   - not yet active: `Semantic PR`_
 
 In order to submit a patch, please follow the steps below:
 
@@ -57,6 +72,9 @@ In order to submit a patch, please follow the steps below:
   - PEP8_ standards apply, except the convention of line length
   - Maximum line length is 120 characters
 
+- The cardinal rule for creating good commits is to ensure there is only one
+  *logical change* per commit / read `Structural split of changes`_
+
 - Check if your code breaks existing tests.  If so, update the tests or fix your
   code.
 
@@ -64,6 +82,16 @@ In order to submit a patch, please follow the steps below:
 
 - Add yourself to the :origin:`AUTHORS.rst` file.
 
+- Choose meaning full commit messages, read `Conventional Commits`_
+
+  .. code::
+
+     <type>[optional scope]: <description>
+
+     [optional body]
+
+     [optional footer(s)]
+
 - Create a pull request.
 
 For more help on getting started with searx development, see :ref:`devquickstart`.
@@ -79,6 +107,8 @@ Translation currently takes place on :ref:`transifex <translation>`.
    Please, do not update translation files in the repo.
 
 
+.. _contrib docs:
+
 Documentation
 =============
 
@@ -91,7 +121,7 @@ Documentation
 
 The documentation is built using Sphinx_.  So in order to be able to generate
 the required files, you have to install it on your system.  Much easier, use
-Makefile our targets.
+our :ref:`makefile`.
 
 Here is an example which makes a complete rebuild:
 
@@ -101,6 +131,7 @@ Here is an example which makes a complete rebuild:
    ...
    The HTML pages are in dist/docs.
 
+.. _make docs-live:
 
 live build
 ----------
@@ -110,9 +141,10 @@ live build
    It is recommended to assert a complete rebuild before deploying (use
    ``docs-clean``).
 
-Live build is like WYSIWYG, If you want to edit the documentation, its
-recommended to use.  The Makefile target ``docs-live`` builds the docs, opens URL
-in your favorite browser and rebuilds every time a reST file has been changed.
+Live build is like WYSIWYG.  If you want to edit the documentation, its
+recommended to use.  The Makefile target ``docs-live`` builds the docs, opens
+URL in your favorite browser and rebuilds every time a reST file has been
+changed.
 
 .. code:: sh
 
@@ -123,12 +155,13 @@ in your favorite browser and rebuilds every time a reST file has been changed.
    ... Start watching changes
 
 
+.. _deploy on github.io:
 
 deploy on github.io
 -------------------
 
 To deploy documentation at :docs:`github.io <.>` use Makefile target
-``gh-pages``, which will builds the documentation, clones searx into a sub
+:ref:`make gh-pages`, which will builds the documentation, clones searx into a sub
 folder ``gh-pages``, cleans it, copies the doc build into and runs all the
 needed git add, commit and push:
 

+ 6 - 0
docs/dev/csv_table.txt

@@ -0,0 +1,6 @@
+stub col row 1, column, "loremLorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
+eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
+voluptua."
+stub col row 1, "At vero eos et accusam et justo duo dolores et ea rebum. Stet clita
+kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.", column
+stub col row 1, column, column

+ 2 - 0
docs/dev/engine_overview.rst

@@ -29,6 +29,7 @@ the ones in the engine file.
 It does not matter if an option is stored in the engine file or in the
 settings.  However, the standard way is the following:
 
+.. _engine file:
 
 engine file
 -----------
@@ -43,6 +44,7 @@ time_range_support      boolean     support search time range
 offline                 boolean     engine runs offline
 ======================= =========== ===========================================
 
+.. _engine settings:
 
 settings.yml
 ------------

+ 3 - 0
docs/dev/hello.dot

@@ -0,0 +1,3 @@
+graph G {
+      Hello -- World
+}

+ 2 - 0
docs/dev/index.rst

@@ -11,3 +11,5 @@ Developer documentation
    search_api
    plugins
    translation
+   makefile
+   reST

+ 221 - 0
docs/dev/makefile.rst

@@ -0,0 +1,221 @@
+.. _makefile:
+
+================
+Makefile Targets
+================
+
+.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
+
+.. sidebar:: build environment
+
+   Before looking deeper at the targets, first read about :ref:`makefile setup`
+   and :ref:`make pyenv`.
+
+With the aim to simplify development cycles, started with :pull:`1756` a
+``Makefile`` based boilerplate was added.  If you are not familiar with
+Makefiles, we recommend to read gnu-make_ introduction.
+
+The usage is simple, just type ``make {target-name}`` to *build* a target.
+Calling the ``help`` target gives a first overview::
+
+  $ make help
+    test      - run developer tests
+    docs      - build documentation
+    docs-live - autobuild HTML documentation while editing
+    run       - run developer instance
+    install   - developer install (./local)
+    uninstall - uninstall (./local)
+    gh-pages  - build docs & deploy on gh-pages branch
+    clean     - drop builds and environments
+    ...
+
+.. contents:: Contents
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+
+.. _makefile setup:
+
+Setup
+=====
+
+.. _git stash: https://git-scm.com/docs/git-stash
+
+The main setup is done in the :origin:`Makefile`::
+
+  export GIT_URL=https://github.com/asciimoo/searx
+  export SEARX_URL=https://searx.me
+  export DOCS_URL=https://asciimoo.github.io/searx
+
+.. sidebar:: fork & upstream
+
+   Commit changes in your (local) branch, fork or whatever, but do not push them
+   upstream / `git stash`_ is your friend.
+
+:GIT_URL: Changes this, to point to your searx fork.
+
+:SEARX_URL: Changes this, to point to your searx instance.
+
+:DOCS_URL: If you host your own (branded) documentation, change this URL.
+
+.. _make pyenv:
+
+Python environment
+==================
+
+.. sidebar:: activate environment
+
+   ``source ./local/py3/bin/activate``
+
+With Makefile we do no longer need to build up the virualenv manually (as
+described in the :ref:`devquickstart` guide).  Jump into your git working tree
+and release a ``make pyenv``:
+
+.. code:: sh
+
+   $ cd ~/searx-clone
+   $ make pyenv
+   PYENV     usage: source ./local/py3/bin/activate
+   ...
+
+With target ``pyenv`` a development environment (aka virtualenv) was build up in
+``./local/py3/``.  To make a *developer install* of searx (:origin:`setup.py`)
+into this environment, use make target ``install``:
+
+.. code:: sh
+
+   $ make install
+   PYENV     usage: source ./local/py3/bin/activate
+   PYENV     using virtualenv from ./local/py3
+   PYENV     install .
+
+You have never to think about intermediate targets like ``pyenv`` or
+``install``, the ``Makefile`` chains them as requisites.  Just run your main
+target.
+
+.. sidebar:: drop environment
+
+   To get rid of the existing environment before re-build use :ref:`clean target
+   <make clean>` first.
+
+If you think, something goes wrong with your ./local environment or you change
+the :origin:`setup.py` file (or the requirements listed in
+:origin:`requirements-dev.txt` and :origin:`requirements.txt`), you have to call
+:ref:`make clean`.
+
+
+.. _make run:
+
+``make run``
+============
+
+To get up a running a developer instance simply call ``make run``.  This enables
+*debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``
+instance, disables *debug* option again and opens the URL in your favorite WEB
+browser (:man:`xdg-open`):
+
+.. code:: sh
+
+  $ make run
+  PYENV     usage: source ./local/py3/bin/activate
+  PYENV     install .
+  ./local/py3/bin/python ./searx/webapp.py
+  ...
+  INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)
+  ...
+
+.. _make clean:
+
+``make clean``
+==============
+
+Drop all intermediate files, all builds, but keep sources untouched.  Includes
+target ``pyclean`` which drops ./local environment.  Before calling ``make
+clean`` stop all processes using :ref:`make pyenv`.
+
+.. code:: sh
+
+   $ make clean
+   CLEAN     pyclean
+   CLEAN     clean
+
+.. _make docs:
+
+``make docs docs-live docs-clean``
+==================================
+
+We describe the usage of the ``doc*`` targets in the :ref:`How to contribute /
+Documentation <contrib docs>` section.  If you want to edit the documentation
+read our :ref:`make docs-live` section.  If you are working in your own brand,
+adjust your :ref:`Makefile setup <makefile setup>`.
+
+
+.. _make gh-pages:
+
+``make gh-pages``
+=================
+
+To deploy on github.io first adjust your :ref:`Makefile setup <makefile
+setup>`.  For any further read :ref:`deploy on github.io`.
+
+.. _make test:
+
+``make test``
+=============
+
+Runs a series of tests: ``test.pep8``, ``test.unit``, ``test.robot`` and does
+additional :ref:`pylint checks <make pylint>`.  You can run tests selective,
+e.g.:
+
+.. code:: sh
+
+  $ make test.pep8 test.unit
+  . ./local/py3/bin/activate; ./manage.sh pep8_check
+  [!] Running pep8 check
+  . ./local/py3/bin/activate; ./manage.sh unit_tests
+  [!] Running unit tests
+
+.. _make pylint:
+
+``make pylint``
+===============
+
+.. _Pylint: https://www.pylint.org/
+
+Before commiting its recommend to do some (more) linting.  Pylint_ is known as
+one of the best source-code, bug and quality checker for the Python programming
+language.  Pylint_ is not yet a quality gate within our searx project (like
+:ref:`test.pep8 <make test>` it is), but Pylint_ can help to improve code
+quality anyway.  The pylint profile we use at searx project is found in
+project's root folder :origin:`.pylintrc`.
+
+Code quality is a ongoing process.  Don't try to fix all messages from Pylint,
+run Pylint and check if your changed lines are bringing up new messages.  If so,
+fix it.  By this, code quality gets incremental better and if there comes the
+day, the linting is balanced out, we might decide to add Pylint as a quality
+gate.
+
+
+``make pybuild``
+================
+
+.. _PyPi: https://pypi.org/
+.. _twine: https://twine.readthedocs.io/en/latest/
+
+Build Python packages in ``./dist/py``.
+
+.. code:: sh
+
+  $ make pybuild
+  ...
+  BUILD     pybuild
+  running sdist
+  running egg_info
+  ...
+  $ ls  ./dist/py/
+  searx-0.15.0-py3-none-any.whl  searx-0.15.0.tar.gz
+
+To upload packages to PyPi_, there is also a ``upload-pypi`` target.  It needs
+twine_ to be installed.  Since you are not the owner of :pypi:`searx` you will
+never need the latter.

+ 6 - 0
docs/dev/plugins.rst

@@ -1,7 +1,13 @@
+.. _dev plugin:
+
 =======
 Plugins
 =======
 
+.. sidebar:: Further reading ..
+
+   - :ref:`plugins generic`
+
 Plugins can extend or replace functionality of various components of searx.
 
 Example plugin

+ 24 - 2
docs/dev/quickstart.rst

@@ -4,15 +4,23 @@
 Development Quickstart
 ======================
 
+.. sidebar:: :ref:`makefile`
+
+   For additional developer purpose there are :ref:`makefile`.
+
 This quickstart guide gets your environment set up with searx.  Furthermore, it
 gives a short introduction to the ``manage.sh`` script.
 
 How to setup your development environment
 =========================================
 
+.. sidebar:: :ref:`make pyenv <make pyenv>`
+
+   Alternatively use the :ref:`make pyenv`.
+
 First, clone the source code of searx to the desired folder.  In this case the
 source is cloned to ``~/myprojects/searx``.  Then create and activate the
-searx-ve virtualenv and install the required packages using manage.sh.
+searx-ve virtualenv and install the required packages using ``manage.sh``.
 
 .. code:: sh
 
@@ -27,6 +35,10 @@ searx-ve virtualenv and install the required packages using manage.sh.
 How to run tests
 ================
 
+.. sidebar:: :ref:`make test.unit <make test>`
+
+   Alternatively use the ``test.pep8``, ``test.unit``, ``test.robot`` targets.
+
 Tests can be run using the ``manage.sh`` script.  Following tests and checks are
 available:
 
@@ -41,7 +53,8 @@ For example unit tests are run with the command below:
 
    ./manage.sh unit_tests
 
-For further test options, please consult the help of the ``manage.sh`` script.
+For further test options, please consult the help of the ``manage.sh`` script or
+read :ref:`make test`.
 
 
 How to compile styles and javascript
@@ -97,6 +110,11 @@ After installing grunt, the files can be built using the following command:
 Tips for debugging/development
 ==============================
 
+.. sidebar:: :ref:`make run`
+
+   Makefile target ``run`` already enables debug option for your developer
+   session / see :ref:`make run`.
+
 Turn on debug logging
   Whether you are working on a new engine or trying to eliminate a bug, it is
   always a good idea to turn on debug logging.  When debug logging is enabled a
@@ -104,6 +122,10 @@ Turn on debug logging
   message. It can be turned on by setting ``debug: False`` to ``debug: True`` in
   :origin:`settings.yml <searx/settings.yml>`.
 
+.. sidebar:: :ref:`make test`
+
+   Alternatively use the :ref:`make test` targets.
+
 Run ``./manage.sh tests`` before creating a PR.
   Failing build on Travis is common because of PEP8 checks.  So a new commit
   must be created containing these format fixes.  This phase can be skipped if

+ 1428 - 0
docs/dev/reST.rst

@@ -0,0 +1,1428 @@
+.. _reST primer:
+
+===========
+reST primer
+===========
+
+.. sidebar:: KISS_ and readability_
+
+   Instead of defining more and more roles, we at searx encourage our
+   contributors to follow principles like KISS_ and readability_.
+
+We at searx are using reStructuredText (aka reST_) markup for all kind of
+documentation, with the builders from the Sphinx_ project a HTML output is
+generated and deployed at :docs:`github.io <.>`.  For build prerequisites read
+:ref:`docs build`.
+
+The source files of Searx's documentation are located at :origin:`docs`.  Sphinx
+assumes source files to be encoded in UTF-8 by defaul.  Run :ref:`make docs-live
+<make docs-live>` to build HTML while editing.
+
+.. sidebar:: Further reading
+
+   - Sphinx-Primer_
+   - `Sphinx markup constructs`_
+   - reST_, docutils_, `docutils FAQ`_
+   - Sphinx_, `sphinx-doc FAQ`_
+   - `sphinx config`_, doctree_
+   - `sphinx cross references`_
+   - linuxdoc_
+   - intersphinx_
+   - sphinx-jinja_
+   - `Sphinx's autodoc`_
+   - `Sphinx's Python domain`_, `Sphinx's C domain`_
+   - SVG_, ImageMagick_
+   - DOT_, `Graphviz's dot`_, Graphviz_
+
+
+.. contents:: Contents
+   :depth: 3
+   :local:
+   :backlinks: entry
+
+Sphinx_ and reST_ have their place in the python ecosystem.  Over that reST is
+used in popular projects, e.g the Linux kernel documentation `[kernel doc]`_.
+
+.. _[kernel doc]: https://www.kernel.org/doc/html/latest/doc-guide/sphinx.html
+
+.. sidebar:: Content matters
+
+   The readability_ of the reST sources has its value, therefore we recommend to
+   make sparse usage of reST markup / .. content matters!
+
+**reST** is a plaintext markup language, its markup is *mostly* intuitive and
+you will not need to learn much to produce well formed articles with.  I use the
+word *mostly*: like everything in live, reST has its advantages and
+disadvantages, some markups feel a bit grumpy (especially if you are used to
+other plaintext markups).
+
+Soft skills
+===========
+
+Before going any deeper into the markup let's face on some **soft skills** a
+trained author brings with, to reach a well feedback from readers:
+
+- Documentation is dedicated to an audience and answers questions from the
+  audience point of view.
+- Don't detail things which are general knowledge from the audience point of
+  view.
+- Limit the subject, use cross links for any further reading.
+
+To be more concrete what a *point of view* means.  In the (:origin:`docs`)
+folder we have three sections (and the *blog* folder), each dedicate to a
+different group of audience.
+
+User's POV: :origin:`docs/user`
+  A typical user knows about search engines and might have heard about
+  meta crawlers and privacy.
+
+Admin's POV: :origin:`docs/admin`
+  A typical Admin knows about setting up services on a linux system, but he does
+  not know all the pros and cons of a searx setup.
+
+Developer's POV: :origin:`docs/dev`
+  Depending on the readability_ of code, a typical developer is able to read and
+  understand source code.  Describe what a item aims to do (e.g. a function).
+  If the chronological order matters, describe it.  Name the *out-of-limits
+  conditions* and all the side effects a external developer will not know.
+
+.. _reST inline markup:
+
+Basic inline markup
+===================
+
+.. sidebar:: Inline markup
+
+   - :ref:`reST roles`
+   - :ref:`reST smart ref`
+
+Basic inline markup is done with asterisks and backquotes.  If asterisks or
+backquotes appear in running text and could be confused with inline markup
+delimiters, they have to be escaped with a backslash (``\*pointer``).
+
+.. table:: basic inline markup
+   :widths: 4 3 7
+
+   ================================================ ==================== ========================
+   description                                      rendered             markup
+   ================================================ ==================== ========================
+   one asterisk for emphasis                        *italics*            ``*italics*``
+   two asterisks for strong emphasis                **boldface**         ``**boldface**``
+   backquotes for code samples and literals         ``foo()``            ````foo()````
+   quote asterisks or backquotes                    \*foo is a pointer   ``\*foo is a pointer``
+   ================================================ ==================== ========================
+
+.. _reST basic structure:
+
+Basic article structure
+=======================
+
+The basic structure of an article makes use of heading adornments to markup
+chapter, sections and subsections.
+
+.. _reST template:
+
+reST template
+-------------
+
+reST template for an simple article:
+
+.. code:: reST
+
+    .. _doc refname:
+
+    ==============
+    Document title
+    ==============
+
+    Lorem ipsum dolor sit amet, consectetur adipisici elit ..  Further read
+    :ref:`chapter refname`.
+
+    .. _chapter refname:
+
+    Chapter
+    =======
+
+    Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut
+    aliquid ex ea commodi consequat ...
+
+    .. _section refname:
+
+    Section
+    -------
+
+    lorem ..
+
+    .. _subsection refname:
+
+    Subsection
+    ~~~~~~~~~~
+
+    lorem ..
+
+
+Headings
+--------
+
+#. title - with overline for document title:
+
+  .. code:: reST
+
+    ==============
+    Document title
+    ==============
+
+
+#. chapter - with anchor named ``anchor name``:
+
+   .. code:: reST
+
+      .. _anchor name:
+
+      Chapter
+      =======
+
+#. section
+
+   .. code:: reST
+
+      Section
+      -------
+
+#. subsection
+
+   .. code:: reST
+
+      Subsection
+      ~~~~~~~~~~
+
+
+
+Anchors & Links
+===============
+
+.. _reST anchor:
+
+Anchors
+-------
+
+.. _ref role:
+   https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
+
+To refer a point in the documentation a anchor is needed.  The :ref:`reST
+template <reST template>` shows an example where a chapter titled *"Chapters"*
+gets an anchor named ``chapter title``.  Another example from *this* document,
+where the anchor named ``reST anchor``:
+
+.. code:: reST
+
+   .. _reST anchor:
+
+   Anchors
+   -------
+
+   To refer a point in the documentation a anchor is needed ...
+
+To refer anchors use the `ref role`_ markup:
+
+.. code:: reST
+
+   Visit chapter :ref:`reST anchor`.  Or set hyperlink text manualy :ref:`foo
+   bar <reST anchor>`.
+
+.. admonition:: ``:ref:`` role
+   :class: rst-example
+
+   Visist chapter :ref:`reST anchor`.  Or set hyperlink text manualy :ref:`foo
+   bar <reST anchor>`.
+
+.. _reST ordinary ref:
+
+Link ordinary URL
+-----------------
+
+If you need to reference external URLs use *named* hyperlinks to maintain
+readability of reST sources.  Here is a example taken from *this* article:
+
+.. code:: reST
+
+   .. _Sphinx Field Lists:
+      https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html
+
+   With the *named* hyperlink `Sphinx Field Lists`_, the raw text is much more
+   readable.
+
+   And this shows the alternative (less readable) hyperlink markup `Sphinx Field
+   Lists
+   <https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html>`__.
+
+.. admonition:: Named hyperlink
+   :class: rst-example
+
+   With the *named* hyperlink `Sphinx Field Lists`_, the raw text is much more
+   readable.
+
+   And this shows the alternative (less readable) hyperlink markup `Sphinx Field
+   Lists
+   <https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html>`__.
+
+
+.. _reST smart ref:
+
+Smart refs
+----------
+
+With the power of sphinx.ext.extlinks_ and intersphinx_ referencing external
+content becomes smart.
+
+.. table:: smart refs with sphinx.ext.extlinks_ and intersphinx_
+   :widths: 4 3 7
+
+   ========================== ================================== ====================================
+   refer ...                  rendered example                   markup
+   ========================== ================================== ====================================
+   :rst:role:`rfc`            :rfc:`822`                         ``:rfc:`822```
+   :rst:role:`pep`            :pep:`8`                           ``:pep:`8```
+   sphinx.ext.extlinks_
+   --------------------------------------------------------------------------------------------------
+   project's wiki article     :wiki:`Searx-instances`            ``:wiki:`Searx-instances```
+   to docs public URL         :docs:`dev/reST.html`              ``:docs:`dev/reST.html```
+   files & folders origin     :origin:`docs/dev/reST.rst`        ``:origin:`docs/dev/reST.rst```
+   pull request               :pull:`1756`                       ``:pull:`1756```
+   patch                      :patch:`af2cae6`                   ``:patch:`af2cae6```
+   PyPi package               :pypi:`searx`                      ``:pypi:`searx```
+   manual page man            :man:`bash`                        ``:man:`bash```
+   intersphinx_
+   --------------------------------------------------------------------------------------------------
+   external anchor            :ref:`python:and`                  ``:ref:`python:and```
+   external doc anchor        :doc:`jinja:templates`             ``:doc:`jinja:templates```
+   python code object         :py:obj:`datetime.datetime`        ``:py:obj:`datetime.datetime```
+   flask code object          :py:obj:`flask.Flask`              ``:py:obj:`flask.Flask```
+   ========================== ================================== ====================================
+
+
+Intersphinx is configured in :origin:`docs/conf.py`:
+
+.. code:: python
+
+    intersphinx_mapping = {
+        "python": ("https://docs.python.org/3/", None),
+        "flask": ("https://flask.palletsprojects.com/", None),
+        "jinja": ("https://jinja.palletsprojects.com/", None),
+        "linuxdoc" : ("https://return42.github.io/linuxdoc/", None),
+        "sphinx" : ("https://www.sphinx-doc.org/en/master/", None),
+    }
+
+
+To list all anchors of the inventory (e.g. ``python``) use:
+
+.. code:: sh
+
+   $ python -m sphinx.ext.intersphinx https://docs.python.org/3/objects.inv
+
+Literal blocks
+==============
+
+The simplest form of :duref:`literal-blocks` is a indented block introduced by
+two colons (``::``).  For highlighting use :dudir:`highlight` or :ref:`reST
+code` directive.  To include literals from external files use directive
+:dudir:`literalinclude`.
+
+.. _reST literal:
+
+``::``
+------
+
+.. code:: reST
+
+   ::
+
+     Literal block
+
+   Lorem ipsum dolor::
+
+     Literal block
+
+   Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
+   eirmod tempor invidunt ut labore ::
+
+     Literal block
+
+.. admonition:: Literal block
+   :class: rst-example
+
+   ::
+
+     Literal block
+
+   Lorem ipsum dolor::
+
+     Literal block
+
+   Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
+   eirmod tempor invidunt ut labore ::
+
+     Literal block
+
+
+.. _reST code:
+
+``code-block``
+--------------
+
+.. _pygments: https://pygments.org/languages/
+
+.. sidebar:: Syntax highlighting
+
+   is handled by pygments_.
+
+The :rst:dir:`code-block` directive is a variant of the :dudir:`code` directive
+with additional options.  To learn more about code literals visit
+:ref:`sphinx:code-examples`.
+
+.. code-block:: reST
+
+   The URL ``/stats`` handle is shown in :ref:`stats-handle`
+
+   .. code-block:: Python
+      :caption: python code block
+      :name: stats-handle
+
+      @app.route('/stats', methods=['GET'])
+      def stats():
+          """Render engine statistics page."""
+          stats = get_engines_stats()
+          return render(
+              'stats.html'
+              , stats = stats )
+
+.. code-block:: reST
+
+.. admonition:: Code block
+   :class: rst-example
+
+   The URL ``/stats`` handle is shown in :ref:`stats-handle`
+
+   .. code-block:: Python
+      :caption: python code block
+      :name: stats-handle
+
+      @app.route('/stats', methods=['GET'])
+      def stats():
+          """Render engine statistics page."""
+          stats = get_engines_stats()
+          return render(
+              'stats.html'
+              , stats = stats )
+
+Unicode substitution
+====================
+
+The :dudir:`unicode directive <unicode-character-codes>` converts Unicode
+character codes (numerical values) to characters.  This directive can only be
+used within a substitution definition.
+
+.. code-block:: reST
+
+   .. |copy| unicode:: 0xA9 .. copyright sign
+   .. |(TM)| unicode:: U+2122
+
+   Trademark |(TM)| and copyright |copy| glyphs.
+
+.. admonition:: Unicode
+   :class: rst-example
+
+   .. |copy| unicode:: 0xA9 .. copyright sign
+   .. |(TM)| unicode:: U+2122
+
+   Trademark |(TM)| and copyright |copy| glyphs.
+
+
+.. _reST roles:
+
+Roles
+=====
+
+.. sidebar:: Further reading
+
+   - `Sphinx Roles`_
+   - :doc:`sphinx:usage/restructuredtext/domains`
+
+
+A *custom interpreted text role* (:duref:`ref <roles>`) is an inline piece of
+explicit markup.  It signifies that that the enclosed text should be interpreted
+in a specific way.
+
+The general markup is one of:
+
+.. code:: reST
+
+   :rolename:`ref-name`
+   :rolename:`ref text <ref-name>`
+
+.. table:: smart refs with sphinx.ext.extlinks_ and intersphinx_
+   :widths: 4 3 7
+
+   ========================== ================================== ====================================
+   role                       rendered example                   markup
+   ========================== ================================== ====================================
+   :rst:role:`guilabel`       :guilabel:`&Cancel`                ``:guilabel:`&Cancel```
+   :rst:role:`kbd`            :kbd:`C-x C-f`                     ``:kbd:`C-x C-f```
+   :rst:role:`menuselection`  :menuselection:`Open --> File`     ``:menuselection:`Open --> File```
+   :rst:role:`download`       :download:`this file <reST.rst>`   ``:download:`this file <reST.rst>```
+   math_                      :math:`a^2 + b^2 = c^2`            ``:math:`a^2 + b^2 = c^2```
+   :rst:role:`ref`            :ref:`svg image example`           ``:ref:`svg image example```
+   :rst:role:`command`        :command:`ls -la`                  ``:command:`ls -la```
+   :durole:`emphasis`         :emphasis:`italic`                 ``:emphasis:`italic```
+   :durole:`strong`           :strong:`bold`                     ``:strong:`bold```
+   :durole:`literal`          :literal:`foo()`                   ``:literal:`foo()```
+   :durole:`subscript`        H\ :sub:`2`\ O                     ``H\ :sub:`2`\ O``
+   :durole:`superscript`      E = mc\ :sup:`2`                   ``E = mc\ :sup:`2```
+   :durole:`title-reference`  :title:`Time`                      ``:title:`Time```
+   ========================== ================================== ====================================
+
+Figures & Images
+================
+
+.. sidebar:: Image processing
+
+   With the directives from :ref:`linuxdoc <linuxdoc:kfigure>` the build process
+   is flexible.  To get best results in the generated output format, install
+   ImageMagick_ and Graphviz_.
+
+Searx's sphinx setup includes: :ref:`linuxdoc:kfigure`.  Scaleable here means;
+scaleable in sense of the build process.  Normally in absence of a converter
+tool, the build process will break.  From the authors POV it’s annoying to care
+about the build process when handling with images, especially since he has no
+access to the build process.  With :ref:`linuxdoc:kfigure` the build process
+continues and scales output quality in dependence of installed image processors.
+
+If you want to add an image, you should use the ``kernel-figure`` (inheritance
+of :dudir:`figure`) and ``kernel-image`` (inheritance of :dudir:`image`)
+directives.  E.g. to insert a figure with a scaleable image format use SVG
+(:ref:`svg image example`):
+
+.. code:: reST
+
+   .. _svg image example:
+
+   .. kernel-figure:: svg_image.svg
+      :alt: SVG image example
+
+      Simple SVG image
+
+    To refer the figure, a caption block is needed: :ref:`svg image example`.
+
+.. _svg image example:
+
+.. kernel-figure:: svg_image.svg
+   :alt: SVG image example
+
+   Simple SVG image.
+
+To refer the figure, a caption block is needed: :ref:`svg image example`.
+
+DOT files (aka Graphviz)
+------------------------
+
+With :ref:`linuxdoc:kernel-figure` reST support for **DOT** formatted files is
+given.
+
+- `Graphviz's dot`_
+- DOT_
+- Graphviz_
+
+A simple example is shown in :ref:`dot file example`:
+
+.. code:: reST
+
+   .. _dot file example:
+
+   .. kernel-figure:: hello.dot
+      :alt: hello world
+
+      DOT's hello world example
+
+.. admonition:: hello.dot
+   :class: rst-example
+
+   .. _dot file example:
+
+   .. kernel-figure:: hello.dot
+      :alt: hello world
+
+      DOT's hello world example
+
+``kernel-render`` DOT
+---------------------
+
+Embed *render* markups (or languages) like Graphviz's **DOT** is provided by the
+:ref:`linuxdoc:kernel-render` directive.  A simple example of embedded DOT_ is
+shown in figure :ref:`dot render example`:
+
+.. code:: reST
+
+   .. _dot render example:
+
+   .. kernel-render:: DOT
+      :alt: digraph
+      :caption: Embedded  DOT (Graphviz) code
+
+      digraph foo {
+        "bar" -> "baz";
+      }
+
+   Attribute ``caption`` is needed, if you want to refer the figure: :ref:`dot
+   render example`.
+
+Please note :ref:`build tools <linuxdoc:kfigure_build_tools>`.  If Graphviz_ is
+installed, you will see an vector image.  If not, the raw markup is inserted as
+*literal-block*.
+
+.. admonition:: kernel-render DOT
+   :class: rst-example
+
+   .. _dot render example:
+
+   .. kernel-render:: DOT
+      :alt: digraph
+      :caption: Embedded  DOT (Graphviz) code
+
+      digraph foo {
+        "bar" -> "baz";
+      }
+
+   Attribute ``caption`` is needed, if you want to refer the figure: :ref:`dot
+   render example`.
+
+``kernel-render`` SVG
+---------------------
+
+A simple example of embedded SVG_ is shown in figure :ref:`svg render example`:
+
+.. code:: reST
+
+   .. _svg render example:
+
+   .. kernel-render:: SVG
+      :caption: Embedded **SVG** markup
+      :alt: so-nw-arrow
+..
+
+  .. code:: xml
+
+      <?xml version="1.0" encoding="UTF-8"?>
+      <svg xmlns="http://www.w3.org/2000/svg" version="1.1"
+           baseProfile="full" width="70px" height="40px"
+           viewBox="0 0 700 400"
+           >
+        <line x1="180" y1="370"
+              x2="500" y2="50"
+              stroke="black" stroke-width="15px"
+              />
+        <polygon points="585 0 525 25 585 50"
+                 transform="rotate(135 525 25)"
+                 />
+      </svg>
+
+.. admonition:: kernel-render SVG
+   :class: rst-example
+
+   .. _svg render example:
+
+   .. kernel-render:: SVG
+      :caption: Embedded **SVG** markup
+      :alt: so-nw-arrow
+
+      <?xml version="1.0" encoding="UTF-8"?>
+      <svg xmlns="http://www.w3.org/2000/svg" version="1.1"
+           baseProfile="full" width="70px" height="40px"
+           viewBox="0 0 700 400"
+           >
+        <line x1="180" y1="370"
+              x2="500" y2="50"
+              stroke="black" stroke-width="15px"
+              />
+        <polygon points="585 0 525 25 585 50"
+                 transform="rotate(135 525 25)"
+                 />
+      </svg>
+
+
+
+
+.. _reST lists:
+
+List markups
+============
+
+Bullet list
+-----------
+
+List markup (:duref:`ref <bullet-lists>`) is simple:
+
+.. code:: reST
+
+   - This is a bulleted list.
+
+     1. Nested lists are possible, but be aware that they must be separated from
+        the parent list items by blank line
+     2. Second item of nested list
+
+   - It has two items, the second
+     item uses two lines.
+
+   #. This is a numbered list.
+   #. It has two items too.
+
+.. admonition:: bullet list
+   :class: rst-example
+
+   - This is a bulleted list.
+
+     1. Nested lists are possible, but be aware that they must be separated from
+        the parent list items by blank line
+     2. Second item of nested list
+
+   - It has two items, the second
+     item uses two lines.
+
+   #. This is a numbered list.
+   #. It has two items too.
+
+
+Horizontal list
+---------------
+
+The :rst:dir:`.. hlist:: <hlist>` transforms a bullet list into a more compact
+list.
+
+.. code:: reST
+
+   .. hlist::
+
+      - first list item
+      - second list item
+      - third list item
+      ...
+
+.. admonition:: hlist
+   :class: rst-example
+
+   .. hlist::
+
+      - first list item
+      - second list item
+      - third list item
+      - next list item
+      - next list item xxxx
+      - next list item yyyy
+      - next list item zzzz
+
+
+Definition list
+---------------
+
+.. sidebar:: Note ..
+
+   - the term cannot have more than one line of text
+
+   - there is **no blank line between term and definition block** // this
+     distinguishes definition lists (:duref:`ref <definition-lists>`) from block
+     quotes (:duref:`ref <block-quotes>`).
+
+Each definition list (:duref:`ref <definition-lists>`) item contains a term,
+optional classifiers and a definition.  A term is a simple one-line word or
+phrase.  Optional classifiers may follow the term on the same line, each after
+an inline ' : ' (**space, colon, space**).  A definition is a block indented
+relative to the term, and may contain multiple paragraphs and other body
+elements.  There may be no blank line between a term line and a definition block
+(*this distinguishes definition lists from block quotes*).  Blank lines are
+required before the first and after the last definition list item, but are
+optional in-between.
+
+Definition lists are created as follows:
+
+.. code:: reST
+
+   term 1 (up to a line of text)
+       Definition 1.
+
+   See the typo : this line is not a term!
+
+     And this is not term's definition.  **There is a blank line** in between
+     the line above and this paragraph.  That's why this paragraph is taken as
+     **block quote** (:duref:`ref <block-quotes>`) and not as term's definition!
+
+   term 2
+       Definition 2, paragraph 1.
+
+       Definition 2, paragraph 2.
+
+   term 3 : classifier
+       Definition 3.
+
+   term 4 : classifier one : classifier two
+       Definition 4.
+
+.. admonition:: definition list
+   :class: rst-example
+
+   term 1 (up to a line of text)
+       Definition 1.
+
+   See the typo : this line is not a term!
+
+     And this is not term's definition.  **There is a blank line** in between
+     the line above and this paragraph.  That's why this paragraph is taken as
+     **block quote** (:duref:`ref <block-quotes>`) and not as term's definition!
+
+
+   term 2
+       Definition 2, paragraph 1.
+
+       Definition 2, paragraph 2.
+
+   term 3 : classifier
+       Definition 3.
+
+   term 4 : classifier one : classifier two
+
+
+Quoted paragraphs
+-----------------
+
+Quoted paragraphs (:duref:`ref <block-quotes>`) are created by just indenting
+them more than the surrounding paragraphs.  Line blocks (:duref:`ref
+<line-blocks>`) are a way of preserving line breaks:
+
+.. code:: reST
+
+   normal paragraph ...
+   lorem ipsum.
+
+      Quoted paragraph ...
+      lorem ipsum.
+
+   | These lines are
+   | broken exactly like in
+   | the source file.
+
+
+.. admonition:: Quoted paragraph and line block
+   :class: rst-example
+
+   normal paragraph ...
+   lorem ipsum.
+
+      Quoted paragraph ...
+      lorem ipsum.
+
+   | These lines are
+   | broken exactly like in
+   | the source file.
+
+
+.. _reST field list:
+
+Field Lists
+-----------
+
+.. _Sphinx Field Lists:
+   https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html
+
+.. sidebar::  bibliographic fields
+
+   First lines fields are bibliographic fields, see `Sphinx Field Lists`_.
+
+Field lists are used as part of an extension syntax, such as options for
+directives, or database-like records meant for further processing.  Field lists
+are mappings from field names to field bodies.  They marked up like this:
+
+.. code:: reST
+
+   :fieldname: Field content
+   :foo:       first paragraph in field foo
+
+               second paragraph in field foo
+
+   :bar:       Field content
+
+.. admonition:: Field List
+   :class: rst-example
+
+   :fieldname: Field content
+   :foo:       first paragraph in field foo
+
+               second paragraph in field foo
+
+   :bar:       Field content
+
+
+They are commonly used in Python documentation:
+
+.. code:: python
+
+   def my_function(my_arg, my_other_arg):
+       """A function just for me.
+
+       :param my_arg: The first of my arguments.
+       :param my_other_arg: The second of my arguments.
+
+       :returns: A message (just for me, of course).
+       """
+
+Further list blocks
+-------------------
+
+- field lists (:duref:`ref <field-lists>`, with caveats noted in
+  :ref:`reST field list`)
+- option lists (:duref:`ref <option-lists>`)
+- quoted literal blocks (:duref:`ref <quoted-literal-blocks>`)
+- doctest blocks (:duref:`ref <doctest-blocks>`)
+
+
+Admonitions
+===========
+
+Sidebar
+-------
+
+Sidebar is an eye catcher, often used for admonitions pointing further stuff or
+site effects.  Here is the source of the sidebar :ref:`on top of this page <reST
+primer>`.
+
+.. code:: reST
+
+   .. sidebar:: KISS_ and readability_
+
+      Instead of defining more and more roles, we at searx encourage our
+      contributors to follow principles like KISS_ and readability_.
+
+Generic admonition
+------------------
+
+The generic :dudir:`admonition <admonitions>` needs a title:
+
+.. code:: reST
+
+   .. admonition:: generic admonition title
+
+      lorem ipsum ..
+
+
+.. admonition:: generic admonition title
+
+   lorem ipsum ..
+
+
+Specific admonitions
+--------------------
+
+Specific admonitions: :dudir:`hint`, :dudir:`note`, :dudir:`tip` :dudir:`attention`,
+:dudir:`caution`, :dudir:`danger`, :dudir:`error`, , :dudir:`important`, and
+:dudir:`warning` .
+
+.. code:: reST
+
+   .. hint::
+
+      lorem ipsum ..
+
+   .. note::
+
+      lorem ipsum ..
+
+   .. warning::
+
+      lorem ipsum ..
+
+
+.. hint::
+
+   lorem ipsum ..
+
+.. note::
+
+   lorem ipsum ..
+
+.. tip::
+
+   lorem ipsum ..
+
+.. attention::
+
+   lorem ipsum ..
+
+.. caution::
+
+   lorem ipsum ..
+
+.. danger::
+
+   lorem ipsum ..
+
+.. important::
+
+   lorem ipsum ..
+
+.. error::
+
+   lorem ipsum ..
+
+.. warning::
+
+   lorem ipsum ..
+
+
+Tables
+======
+
+.. sidebar:: Nested tables
+
+   Nested tables are ugly! Not all builder support nested tables, don't use
+   them!
+
+ASCII-art tables like :ref:`reST simple table` and :ref:`reST grid table` might
+be comfortable for readers of the text-files, but they have huge disadvantages
+in the creation and modifying.  First, they are hard to edit.  Think about
+adding a row or a column to a ASCII-art table or adding a paragraph in a cell,
+it is a nightmare on big tables.
+
+
+.. sidebar:: List tables
+
+   For meaningful patch and diff use :ref:`reST flat table`.
+
+Second the diff of modifying ASCII-art tables is not meaningful, e.g. widening a
+cell generates a diff in which also changes are included, which are only
+ascribable to the ASCII-art.  Anyway, if you prefer ASCII-art for any reason,
+here are some helpers:
+
+* `Emacs Table Mode`_
+* `Online Tables Generator`_
+
+.. _reST simple table:
+
+Simple tables
+-------------
+
+:duref:`Simple tables <simple-tables>` allow *colspan* but not *rowspan*.  If
+your table need some metadata (e.g. a title) you need to add the ``.. table::
+directive`` :dudir:`(ref) <table>` in front and place the table in its body:
+
+.. code:: reST
+
+   .. table:: foo gate truth table
+      :widths: grid
+      :align: left
+
+      ====== ====== ======
+          Inputs    Output
+      ------------- ------
+      A      B      A or B
+      ====== ====== ======
+      False
+      --------------------
+      True
+      --------------------
+      True   False  True
+             (foo)
+      ------ ------ ------
+      False  True
+             (foo)
+      ====== =============
+
+.. admonition:: Simple ASCII table
+   :class: rst-example
+
+   .. table:: foo gate truth table
+      :widths: grid
+      :align: left
+
+      ====== ====== ======
+          Inputs    Output
+      ------------- ------
+      A      B      A or B
+      ====== ====== ======
+      False
+      --------------------
+      True
+      --------------------
+      True   False  True
+             (foo)
+      ------ ------ ------
+      False  True
+             (foo)
+      ====== =============
+
+
+
+.. _reST grid table:
+
+Grid tables
+-----------
+
+:duref:`Grid tables <grid-tables>` allow colspan *colspan* and *rowspan*:
+
+.. code:: reST
+
+   .. table:: grid table example
+      :widths: 1 1 5
+
+      +------------+------------+-----------+
+      | Header 1   | Header 2   | Header 3  |
+      +============+============+===========+
+      | body row 1 | column 2   | column 3  |
+      +------------+------------+-----------+
+      | body row 2 | Cells may span columns.|
+      +------------+------------+-----------+
+      | body row 3 | Cells may  | - Cells   |
+      +------------+ span rows. | - contain |
+      | body row 4 |            | - blocks. |
+      +------------+------------+-----------+
+
+.. admonition:: ASCII grid table
+   :class: rst-example
+
+   .. table:: grid table example
+      :widths: 1 1 5
+
+      +------------+------------+-----------+
+      | Header 1   | Header 2   | Header 3  |
+      +============+============+===========+
+      | body row 1 | column 2   | column 3  |
+      +------------+------------+-----------+
+      | body row 2 | Cells may span columns.|
+      +------------+------------+-----------+
+      | body row 3 | Cells may  | - Cells   |
+      +------------+ span rows. | - contain |
+      | body row 4 |            | - blocks. |
+      +------------+------------+-----------+
+
+
+.. _reST flat table:
+
+flat-table
+----------
+
+The ``flat-table`` is a further developed variant of the :ref:`list tables
+<linuxdoc:list-table-directives>`.  It is a double-stage list similar to the
+:dudir:`list-table` with some additional features:
+
+column-span: ``cspan``
+  with the role ``cspan`` a cell can be extended through additional columns
+
+row-span: ``rspan``
+  with the role ``rspan`` a cell can be extended through additional rows
+
+auto-span:
+  spans rightmost cell of a table row over the missing cells on the right side
+  of that table-row.  With Option ``:fill-cells:`` this behavior can changed
+  from *auto span* to *auto fill*, which automatically inserts (empty) cells
+  instead of spanning the last cell.
+
+options:
+  :header-rows:   [int] count of header rows
+  :stub-columns:  [int] count of stub columns
+  :widths:        [[int] [int] ... ] widths of columns
+  :fill-cells:    instead of auto-span missing cells, insert missing cells
+
+roles:
+  :cspan: [int] additional columns (*morecols*)
+  :rspan: [int] additional rows (*morerows*)
+
+The example below shows how to use this markup.  The first level of the staged
+list is the *table-row*. In the *table-row* there is only one markup allowed,
+the list of the cells in this *table-row*. Exception are *comments* ( ``..`` )
+and *targets* (e.g. a ref to :ref:`row 2 of table's body <row body 2>`).
+
+.. code:: reST
+
+   .. flat-table:: ``flat-table`` example
+      :header-rows: 2
+      :stub-columns: 1
+      :widths: 1 1 1 1 2
+
+      * - :rspan:`1` head / stub
+        - :cspan:`3` head 1.1-4
+
+      * - head 2.1
+        - head 2.2
+        - head 2.3
+        - head 2.4
+
+      * .. row body 1 / this is a comment
+
+        - row 1
+        - :rspan:`2` cell 1-3.1
+        - cell 1.2
+        - cell 1.3
+        - cell 1.4
+
+      * .. Comments and targets are allowed on *table-row* stage.
+        .. _`row body 2`:
+
+        - row 2
+        - cell 2.2
+        - :rspan:`1` :cspan:`1`
+          cell 2.3 with a span over
+
+          * col 3-4 &
+          * row 2-3
+
+      * - row 3
+        - cell 3.2
+
+      * - row 4
+        - cell 4.1
+        - cell 4.2
+        - cell 4.3
+        - cell 4.4
+
+      * - row 5
+        - cell 5.1 with automatic span to rigth end
+
+      * - row 6
+        - cell 6.1
+        - ..
+
+
+.. admonition:: List table
+   :class: rst-example
+
+   .. flat-table:: ``flat-table`` example
+      :header-rows: 2
+      :stub-columns: 1
+      :widths: 1 1 1 1 2
+
+      * - :rspan:`1` head / stub
+        - :cspan:`3` head 1.1-4
+
+      * - head 2.1
+        - head 2.2
+        - head 2.3
+        - head 2.4
+
+      * .. row body 1 / this is a comment
+
+        - row 1
+        - :rspan:`2` cell 1-3.1
+        - cell 1.2
+        - cell 1.3
+        - cell 1.4
+
+      * .. Comments and targets are allowed on *table-row* stage.
+        .. _`row body 2`:
+
+        - row 2
+        - cell 2.2
+        - :rspan:`1` :cspan:`1`
+          cell 2.3 with a span over
+
+          * col 3-4 &
+          * row 2-3
+
+      * - row 3
+        - cell 3.2
+
+      * - row 4
+        - cell 4.1
+        - cell 4.2
+        - cell 4.3
+        - cell 4.4
+
+      * - row 5
+        - cell 5.1 with automatic span to rigth end
+
+      * - row 6
+        - cell 6.1
+        - ..
+
+
+CSV table
+---------
+
+CSV table might be the choice if you want to include CSV-data from a outstanding
+(build) process into your documentation.
+
+.. code:: reST
+
+   .. csv-table:: CSV table example
+      :header: .. , Column 1, Column 2
+      :widths: 2 5 5
+      :stub-columns: 1
+      :file: csv_table.txt
+
+Content of file ``csv_table.txt``:
+
+.. literalinclude:: csv_table.txt
+
+.. admonition:: CSV table
+   :class: rst-example
+
+   .. csv-table:: CSV table example
+      :header: .. , Column 1, Column 2
+      :widths: 3 5 5
+      :stub-columns: 1
+      :file: csv_table.txt
+
+Templating
+==========
+
+.. sidebar:: Build environment
+
+   All *generic-doc* tasks are running in the :ref:`build environment <make
+   pyenv>`.
+
+Templating is suitable for documentation which is created generic at the build
+time.  The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`build
+environment <make pyenv>` (with searx modules installed).  We use this e.g. to
+build chapter: :ref:`engines generic`.  Below the jinja directive from the
+:origin:`docs/admin/engines.rst` is shown:
+
+.. literalinclude:: ../admin/engines.rst
+   :language: reST
+   :start-after: .. _configured engines:
+
+The context for the template is selected in the line ``.. jinja:: webapp``.  In
+sphinx's build configuration (:origin:`docs/conf.py`) the ``webapp`` context
+points to the name space of the python module: ``webapp``.
+
+.. code:: py
+
+   from searx import webapp
+   jinja_contexts = {
+       'webapp': dict(**webapp.__dict__)
+   }
+
+
+Tabbed views
+============
+
+.. _sphinx-tabs: https://github.com/djungelorm/sphinx-tabs
+.. _basic-tabs: https://github.com/djungelorm/sphinx-tabs#basic-tabs
+.. _group-tabs: https://github.com/djungelorm/sphinx-tabs#group-tabs
+.. _code-tabs: https://github.com/djungelorm/sphinx-tabs#code-tabs
+
+With `sphinx-tabs`_ extension we have *tabbed views*.  To provide installation
+instructions with one tab per distribution we use the `group-tabs`_ directive,
+others are basic-tabs_ and code-tabs_.  Below a *group-tab* example from
+:ref:`docs build` is shown:
+
+.. literalinclude:: ../admin/buildhosts.rst
+   :language: reST
+   :start-after: .. _system requirements:
+   :end-before: .. _system requirements END:
+
+
+.. _math:
+
+Math equations
+==============
+
+.. _Mathematics: https://en.wikibooks.org/wiki/LaTeX/Mathematics
+.. _amsmath user guide:
+   http://vesta.informatik.rwth-aachen.de/ftp/pub/mirror/ctan/macros/latex/required/amsmath/amsldoc.pdf
+
+.. sidebar:: About LaTeX
+
+   - `amsmath user guide`_
+   - Mathematics_
+   - :ref:`docs build`
+
+The input language for mathematics is LaTeX markup using the :ctan:`amsmath`
+package.
+
+To embed LaTeX markup in reST documents, use role :rst:role:`:math: <math>` for
+inline and directive :rst:dir:`.. math:: <math>` for block markup.
+
+.. code:: reST
+
+   In :math:numref:`schroedinger general` the time-dependent Schrödinger equation
+   is shown.
+
+   .. math::
+      :label: schroedinger general
+
+       \mathrm{i}\hbar\dfrac{\partial}{\partial t} |\,\psi (t) \rangle =
+             \hat{H} |\,\psi (t) \rangle.
+
+.. admonition:: LaTeX math equation
+   :class: rst-example
+
+   In :math:numref:`schroedinger general` the time-dependent Schrödinger equation
+   is shown.
+
+   .. math::
+      :label: schroedinger general
+
+       \mathrm{i}\hbar\dfrac{\partial}{\partial t} |\,\psi (t) \rangle =
+             \hat{H} |\,\psi (t) \rangle.
+
+
+The next example shows the difference of ``\tfrac`` (*textstyle*) and ``\dfrac``
+(*displaystyle*) used in a inline markup or another fraction.
+
+.. code:: reST
+
+   ``\tfrac`` **inline example** :math:`\tfrac{\tfrac{1}{x}+\tfrac{1}{y}}{y-z}`
+   ``\dfrac`` **inline example** :math:`\dfrac{\dfrac{1}{x}+\dfrac{1}{y}}{y-z}`
+
+.. admonition:: Line spacing
+   :class: rst-example
+
+   Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
+   eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
+   voluptua.  ...
+   ``\tfrac`` **inline example** :math:`\tfrac{\tfrac{1}{x}+\tfrac{1}{y}}{y-z}`
+   At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd
+   gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+
+   Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
+   eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
+   voluptua.  ...
+   ``\tfrac`` **inline example** :math:`\dfrac{\dfrac{1}{x}+\dfrac{1}{y}}{y-z}`
+   At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd
+   gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+
+
+.. _KISS: https://en.wikipedia.org/wiki/KISS_principle
+
+.. _readability: https://docs.python-guide.org/writing/style/
+.. _Sphinx-Primer:
+    http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
+.. _reST: https://docutils.sourceforge.io/rst.html
+.. _Sphinx Roles:
+    https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
+.. _Sphinx: http://www.sphinx-doc.org
+.. _`sphinx-doc FAQ`: http://www.sphinx-doc.org/en/stable/faq.html
+.. _Sphinx markup constructs:
+    http://www.sphinx-doc.org/en/stable/markup/index.html
+.. _`sphinx cross references`:
+    http://www.sphinx-doc.org/en/stable/markup/inline.html#cross-referencing-arbitrary-locations
+.. _sphinx.ext.extlinks:
+    https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html
+.. _intersphinx: http://www.sphinx-doc.org/en/stable/ext/intersphinx.html
+.. _sphinx config: http://www.sphinx-doc.org/en/stable/config.html
+.. _Sphinx's autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
+.. _Sphinx's Python domain:
+    http://www.sphinx-doc.org/en/stable/domains.html#the-python-domain
+.. _Sphinx's C domain:
+   http://www.sphinx-doc.org/en/stable/domains.html#cross-referencing-c-constructs
+.. _doctree:
+    http://www.sphinx-doc.org/en/master/extdev/tutorial.html?highlight=doctree#build-phases
+.. _docutils: http://docutils.sourceforge.net/docs/index.html
+.. _docutils FAQ: http://docutils.sourceforge.net/FAQ.html
+.. _linuxdoc: https://return42.github.io/linuxdoc
+.. _jinja: https://jinja.palletsprojects.com/
+.. _sphinx-jinja: https://github.com/tardyp/sphinx-jinja
+.. _SVG: https://www.w3.org/TR/SVG11/expanded-toc.html
+.. _DOT: https://graphviz.gitlab.io/_pages/doc/info/lang.html
+.. _`Graphviz's dot`: https://graphviz.gitlab.io/_pages/pdf/dotguide.pdf
+.. _Graphviz: https://graphviz.gitlab.io
+.. _ImageMagick: https://www.imagemagick.org
+
+.. _`Emacs Table Mode`: https://www.emacswiki.org/emacs/TableMode
+.. _`Online Tables Generator`: http://www.tablesgenerator.com/text_tables
+.. _`OASIS XML Exchange Table Model`: https://www.oasis-open.org/specs/tm9901.html

+ 7 - 0
docs/dev/search_api.rst

@@ -14,6 +14,13 @@ Furthermore, two enpoints ``/`` and ``/search`` are available for querying.
 Parameters
 ==========
 
+.. sidebar:: Further reading ..
+
+   - :ref:`engines generic`
+   - :ref:`configured engines`
+   - :ref:`engine settings`
+   - :ref:`engine file`
+
 ``q`` : required
   The search query.  This string is passed to external search services.  Thus,
   searx supports syntax of each search service.  For example, ``site:github.com

+ 10 - 0
docs/dev/svg_image.svg

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- originate: https://commons.wikimedia.org/wiki/File:Variable_Resistor.svg -->
+<svg xmlns="http://www.w3.org/2000/svg"
+	version="1.1" baseProfile="full"
+	width="70px" height="40px" viewBox="0 0 700 400">
+	<line x1="0" y1="200" x2="700" y2="200" stroke="black" stroke-width="20px"/>
+	<rect x="100" y="100" width="500" height="200" fill="white" stroke="black" stroke-width="20px"/>
+	<line x1="180" y1="370" x2="500" y2="50" stroke="black" stroke-width="15px"/>
+	<polygon points="585 0 525 25 585 50" transform="rotate(135 525 25)"/>
+</svg>

+ 3 - 0
requirements-dev.txt

@@ -11,3 +11,6 @@ transifex-client==0.12.2
 unittest2==1.1.0
 zope.testrunner==4.5.1
 selenium==3.141.0
+linuxdoc @ git+http://github.com/return42/linuxdoc.git
+sphinx-jinja
+sphinx-tabs

+ 2 - 3
searx/engines/bing.py

@@ -89,8 +89,7 @@ def response(resp):
                         'content': content})
 
     try:
-        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))
-        result_len_container = utils.to_string(result_len_container)
+        result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
         if "-" in result_len_container:
             # Remove the part "from-to" for paginated request ...
             result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
@@ -102,7 +101,7 @@ def response(resp):
         logger.debug('result error :\n%s', e)
         pass
 
-    if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
+    if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
         return []
 
     results.append({'number_of_results': result_len})

+ 17 - 9
searx/engines/flickr_noapi.py

@@ -109,14 +109,22 @@ def response(resp):
         else:
             url = build_flickr_url(photo['ownerNsid'], photo['id'])
 
-        results.append({'url': url,
-                        'title': title,
-                        'img_src': img_src,
-                        'thumbnail_src': thumbnail_src,
-                        'content': content,
-                        'author': author,
-                        'source': source,
-                        'img_format': img_format,
-                        'template': 'images.html'})
+        result = {
+            'url': url,
+            'img_src': img_src,
+            'thumbnail_src': thumbnail_src,
+            'source': source,
+            'img_format': img_format,
+            'template': 'images.html'
+        }
+        try:
+            result['author'] = author.encode('utf-8')
+            result['title'] = title.encode('utf-8')
+            result['content'] = content.encode('utf-8')
+        except:
+            result['author'] = ''
+            result['title'] = ''
+            result['content'] = ''
+        results.append(result)
 
     return results

+ 6 - 3
searx/engines/ina.py

@@ -32,7 +32,7 @@ base_url = 'https://www.ina.fr'
 search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}'
 
 # specific xpath variables
-results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]'
+results_xpath = '//div[contains(@class,"search-results--list")]//div[@class="media-body"]'
 url_xpath = './/a/@href'
 title_xpath = './/h3[@class="h3--title media-heading"]'
 thumbnail_xpath = './/img/@src'
@@ -65,8 +65,11 @@ def response(resp):
         videoid = result.xpath(url_xpath)[0]
         url = base_url + videoid
         title = p.unescape(extract_text(result.xpath(title_xpath)))
-        thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
-        if thumbnail[0] == '/':
+        try:
+            thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
+        except:
+            thumbnail = ''
+        if thumbnail and thumbnail[0] == '/':
             thumbnail = base_url + thumbnail
         d = extract_text(result.xpath(publishedDate_xpath)[0])
         d = d.split('/')

+ 2 - 0
searx/engines/microsoft_academic.py

@@ -45,6 +45,8 @@ def request(query, params):
 def response(resp):
     results = []
     response_data = loads(resp.text)
+    if not response_data:
+        return results
 
     for result in response_data['results']:
         url = _get_url(result)

+ 1 - 1
searx/engines/scanr_structures.py

@@ -29,7 +29,7 @@ def request(query, params):
     params['url'] = search_url
     params['method'] = 'POST'
     params['headers']['Content-type'] = "application/json"
-    params['data'] = dumps({"query": query,
+    params['data'] = dumps({"query": query.decode('utf-8'),
                             "searchField": "ALL",
                             "sortDirection": "ASC",
                             "sortOrder": "RELEVANCY",

+ 14 - 0
searx/engines/spotify.py

@@ -12,10 +12,14 @@
 
 from json import loads
 from searx.url_utils import urlencode
+import requests
+import base64
 
 # engine dependent config
 categories = ['music']
 paging = True
+api_client_id = None
+api_client_secret = None
 
 # search-url
 url = 'https://api.spotify.com/'
@@ -31,6 +35,16 @@ def request(query, params):
 
     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
+    r = requests.post(
+        'https://accounts.spotify.com/api/token',
+        data={'grant_type': 'client_credentials'},
+        headers={'Authorization': 'Basic ' + base64.b64encode(
+            "{}:{}".format(api_client_id, api_client_secret).encode('utf-8')
+        ).decode('utf-8')}
+    )
+    j = loads(r.text)
+    params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))}
+
     return params
 
 

+ 20 - 15
searx/settings.yml

@@ -79,9 +79,10 @@ engines:
     categories : science
     timeout : 4.0
 
-  - name : base
-    engine : base
-    shortcut : bs
+#  tmp suspended:  dh key too small
+#  - name : base
+#    engine : base
+#    shortcut : bs
 
   - name : wikipedia
     engine : wikipedia
@@ -408,7 +409,7 @@ engines:
 
   - name : library genesis
     engine : xpath
-    search_url : http://libgen.io/search.php?req={query}
+    search_url : https://libgen.is/search.php?req={query}
     url_xpath : //a[contains(@href,"bookfi.net")]/@href
     title_xpath : //a[contains(@href,"book/")]/text()[1]
     content_xpath : //td/a[1][contains(@href,"=author")]/text()
@@ -464,7 +465,7 @@ engines:
   - name : openairedatasets
     engine : json_engine
     paging : True
-    search_url : http://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query}
+    search_url : https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query}
     results_query : response/results/result
     url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
     title_query : metadata/oaf:entity/oaf:result/title/$
@@ -476,7 +477,7 @@ engines:
   - name : openairepublications
     engine : json_engine
     paging : True
-    search_url : http://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query}
+    search_url : https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query}
     results_query : response/results/result
     url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
     title_query : metadata/oaf:entity/oaf:result/title/$
@@ -552,10 +553,11 @@ engines:
     timeout : 10.0
     disabled : True
 
-  - name : scanr structures
-    shortcut: scs
-    engine : scanr_structures
-    disabled : True
+# tmp suspended: bad certificate
+#  - name : scanr structures
+#    shortcut: scs
+#    engine : scanr_structures
+#    disabled : True
 
   - name : soundcloud
     engine : soundcloud
@@ -598,9 +600,12 @@ engines:
     shortcut : se
     categories : science
 
-  - name : spotify
-    engine : spotify
-    shortcut : stf
+# Spotify needs API credentials
+#  - name : spotify
+#    engine : spotify
+#    shortcut : stf
+#    api_client_id : *******
+#    api_client_secret : *******
 
   - name : startpage
     engine : startpage
@@ -812,7 +817,7 @@ locales:
 doi_resolvers :
   oadoi.org : 'https://oadoi.org/'
   doi.org : 'https://doi.org/'
-  doai.io  : 'http://doai.io/'
-  sci-hub.tw : 'http://sci-hub.tw/'
+  doai.io  : 'https://doai.io/'
+  sci-hub.tw : 'https://sci-hub.tw/'
 
 default_doi_resolver : 'oadoi.org'

+ 2 - 2
searx/settings_robot.yml

@@ -43,7 +43,7 @@ locales:
 doi_resolvers :
   oadoi.org : 'https://oadoi.org/'
   doi.org : 'https://doi.org/'
-  doai.io  : 'http://doai.io/'
-  sci-hub.tw : 'http://sci-hub.tw/'
+  doai.io  : 'https://doai.io/'
+  sci-hub.tw : 'https://sci-hub.tw/'
 
 default_doi_resolver : 'oadoi.org'

+ 10 - 2
searx/static/plugins/js/vim_hotkeys.js

@@ -125,6 +125,14 @@ $(document).ready(function() {
         }
     });
 
+    function nextResult(current, direction) {
+        var next = current[direction]();
+        while (!next.is('.result') && next.length !== 0) {
+            next = next[direction]();
+        }
+        return next
+    }
+
     function highlightResult(which) {
         return function() {
             var current = $('.result[data-vim-selected]');
@@ -157,13 +165,13 @@ $(document).ready(function() {
                         }
                         break;
                     case 'down':
-                        next = current.next('.result');
+                        next = nextResult(current, 'next');
                         if (next.length === 0) {
                             next = $('.result:first');
                         }
                         break;
                     case 'up':
-                        next = current.prev('.result');
+                        next = nextResult(current, 'prev');
                         if (next.length === 0) {
                             next = $('.result:last');
                         }

+ 1 - 0
utils/makefile.sphinx

@@ -198,6 +198,7 @@ $(GH_PAGES)::
 	$(MAKE) docs
 	[ -d "gh-pages/.git" ] || git clone $(GIT_URL) gh-pages
 	-cd $(GH_PAGES); git checkout gh-pages >/dev/null
+	-cd $(GH_PAGES); git pull
 	-cd $(GH_PAGES); ls -A | grep -v '.git$$' | xargs rm -rf
 	cp -r $(DOCS_DIST)/* $(GH_PAGES)/
 	touch $(GH_PAGES)/.nojekyll