utils/searx.sh: add script to install isolated searx service (WIP)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

utils/filtron.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh -*-
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
 # shellcheck disable=SC2119
 
 # shellcheck source=utils/lib.sh

utils/lib.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh -*-
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
 # shellcheck disable=SC2059,SC1117,SC2162,SC2004
 
 ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
@@ -265,9 +265,9 @@ choose_one() {
         _t=""
         err_msg "invalid choice"
     done
+    eval "$env_name"='${list[${REPLY}]}'
     echo
     clean_stdin
-    eval "$env_name"='${list[${REPLY}]}'
 }
 
 install_template() {
@@ -375,34 +375,32 @@ uWSGI_restart() {
 
 uWSGI_install_app() {
 
-    # usage:  uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ...
+    # usage:  uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini
 
-    local do_eval=""
+    local no_eval=""
-    local CONF
+    local CONF=""
 
     if [[ "$1" == "--no-eval" ]]; then
         no_eval=$1; shift
     fi
 
-    for CONF in "$@"; do
+    CONF=$1
-        install_template "$no_eval" "${CONF}" root root 644
+    # shellcheck disable=SC2086
-        uWSGI_enable_app "$(basename "${CONF}")"
+    install_template $no_eval "${CONF}" root root 644
-        info_msg "enabled uWSGI app: $(basename "${CONF}")"
+    uWSGI_enable_app "$(basename "${CONF}")"
-    done
     uWSGI_restart
+    info_msg "installed uWSGI app: $(basename "${CONF}")"
 }
 
 uWSGI_remove_app() {
 
-    # usage:  uWSGI_remove_app <path.ini> ...
+    # usage:  uWSGI_remove_app <path.ini>
 
-    local CONF
+    local CONF=$1
-    for CONF in "$@"; do
+    uWSGI_disable_app "$(basename "${CONF}")"
-        uWSGI_disable_app "$(basename "${CONF}")"
-        rm -f "$CONF"
-        info_msg "removed uWSGI app: $(basename "${CONF}")"
-    done
     uWSGI_restart
+    rm -f "$CONF"
+    info_msg "removed uWSGI app: $(basename "${CONF}")"
 }
 
 # shellcheck disable=SC2164
@@ -416,6 +414,7 @@ uWSGI_enable_app() {
         return 42
     fi
     pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null
+    rm -f "$(basename "${CONF}")"
     # shellcheck disable=SC2226
     ln -s "../apps-available/$(basename "${CONF}")"
     info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"
@@ -454,7 +453,6 @@ pkg_install() {
     fi
     # shellcheck disable=SC2068
     apt-get install -y $@
-    wait_key 30
 }
 
 pkg_remove() {
@@ -468,7 +466,6 @@ pkg_remove() {
         return 42
     fi
     apt-get purge --autoremove --ignore-missing -y "$@"
-    wait_key 30
 }
 
 pkg_is_installed() {
@@ -491,7 +488,7 @@ git_clone() {
     #    git_clone <url> <path> [<branch> [<user>]]
     #
     #  First form uses $CACHE/<name> as destination folder, second form clones
-    #  into <path>.  If repository is allready cloned, merge from origin and
+    #  into <path>.  If repository is allready cloned, pull from <branch> and
     #  update working tree (if needed, the caller has to stash local changes).
     #
     #    git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin
@@ -501,7 +498,8 @@ git_clone() {
     local dest="$2"
     local branch="$3"
     local user="$4"
-    local prefix=""
+    local bash_cmd="bash"
+    local remote="origin"
 
     if [[ ! "${dest:0:1}" = "/" ]]; then
         dest="$CACHE/$dest"
@@ -509,20 +507,21 @@ git_clone() {
 
     [[ -z $branch ]] && branch=master
     [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}"
-    [[ -z $user ]] && prefix="sudo -H -u $user"
+    [[ ! -z $user ]] && bash_cmd="sudo -H -u $user -i"
 
     if [[ -d "${dest}" ]] ; then
         info_msg "already cloned: $dest"
-        pushd "${dest}" > /dev/null
+	tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 |  prefix_stdout "  |$user| "
-        $prefix git checkout -b "$(basename "$branch")" --track "$branch"
+cd "${dest}"
-        $prefix git pull --all
+git checkout -m -B "$branch" --track "$remote/$branch"
-        popd > /dev/null
+git pull --all
-
+EOF
     else
         info_msg "clone into: $dest"
-        $prefix mkdir -p "$(dirname "$dest")"
+	tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 |  prefix_stdout "  |$user| "
-        pushd "${dest}" > /dev/null
+mkdir -p "$(dirname "$dest")"
-        git clone "$url" "$(basename "$dest")"
+cd "$(dirname "$dest")"
-        popd > /dev/null
+git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")"
+EOF
     fi
 }

utils/searx.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh -*-
+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
 # shellcheck disable=SC2119
 
 # shellcheck source=utils/lib.sh
@@ -15,8 +15,11 @@ SERVICE_USER="${SERVICE_NAME}"
 SERVICE_GROUP="${SERVICE_USER}"
 SERVICE_HOME="/home/${SERVICE_USER}"
 
+# shellcheck disable=SC2034
+SEARX_URL="127.0.0.1:8888"
+
 SEARX_GIT_URL="https://github.com/asciimoo/searx.git"
-SEARX_GIT_BRANCH="origin/master"
+SEARX_GIT_BRANCH="master"
 
 # FIXME: Arch Linux & RHEL should be added
 
@@ -25,8 +28,8 @@ libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \
   git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \
   libffi-dev libssl-dev"
 
-SEARX_VENV="${SEARX_HOME}/searx-venv"
+SEARX_PYENV="${SERVICE_HOME}/searx-pyenv"
-SEARX_SRC="${SEARX_HOME}/searx-src"
+SEARX_SRC="${SERVICE_HOME}/searx-src"
 SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml"
 SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}"
 SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini"
@@ -51,25 +54,26 @@ usage(){
 usage:
 
   $(basename "$0") shell
-  $(basename "$0") install    [all|user]
+  $(basename "$0") install    [all|user|pyenv|searx-src]
   $(basename "$0") update     [searx]
-  $(basename "$0") remove     [all]
+  $(basename "$0") remove     [all|user|pyenv|searx-src]
   $(basename "$0") activate   [service]
   $(basename "$0") deactivate [service]
   $(basename "$0") show       [service]
 
 shell
   start interactive shell from user ${SERVICE_USER}
-install / remove all
+install / remove
-  complete setup of searx service
+  all:        complete (de-) installation of searx service
+  user:       add/remove service user '$SERVICE_USER' at $SERVICE_HOME
+  searx-src:  clone $SEARX_GIT_URL
+  pyenv:       create/remove virtualenv (python) in $SEARX_PYENV
 update searx
   Update searx installation of user ${SERVICE_USER}
 activate
   activate and start service daemon (systemd unit)
 deactivate service
   stop and deactivate service daemon (systemd unit)
-install user
-  add service user '$SERVICE_USER' at $SERVICE_HOME
 show service
   show service status and log
 EOF
@@ -102,6 +106,8 @@ main(){
             case $2 in
                 all) install_all ;;
                 user) assert_user ;;
+                pyenv) create_pyenv ;;
+                searx-src) clone_searx ;;
                 *) usage "$_usage"; exit 42;;
             esac ;;
         update)
@@ -115,6 +121,8 @@ main(){
             case $2 in
                 all) remove_all;;
                 user) remove_user ;;
+                pyenv) remove_pyenv ;;
+                searx-src) remove_searx ;;
                 *) usage "$_usage"; exit 42;;
             esac ;;
         activate)
@@ -143,7 +151,7 @@ install_all() {
     wait_key
     clone_searx
     wait_key
-    create_venv
+    create_pyenv
     wait_key
     configure_searx
     wait_key
@@ -167,7 +175,7 @@ update_searx() {
 cd ${SEARX_SRC}
 cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup
 git stash push -m "BACKUP -- 'update server' at ($(date))"
-git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH"
+git checkout -b $SEARX_GIT_BRANCH" --track "$SEARX_GIT_BRANCH"
 git pull "$SEARX_GIT_BRANCH"
 ${SEARX_SRC}/manage.sh update_packages
 EOF
@@ -185,16 +193,16 @@ EOF
            "start interactiv shell"
     case $action in
         "keep new configuration")
-	    info_msg "continue using new settings file"
+            info_msg "continue using new settings file"
-	    ;;
+            ;;
         "revert to the old configuration (backup file)")
     tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
 cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS}
 EOF
-	    ;;
+            ;;
-	"start interactiv shell")
+        "start interactiv shell")
-	    interactive_shell
+            interactive_shell
-	    ;;
+            ;;
     esac
     chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}"
 
@@ -208,7 +216,10 @@ EOF
 
 remove_all() {
     rst_title "De-Install $SERVICE_NAME (service)"
-    remove_service
+    if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then
+        return
+    fi
+    remove_searx_uwsgi
     wait_key
     remove_user
 }
@@ -240,35 +251,71 @@ remove_user() {
 clone_searx(){
     rst_title "Clone searx sources" section
     echo
+    SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)"
+    if [[ ! "${SERVICE_HOME}" ]]; then
+        err_msg "to clone searx sources, user $SERVICE_USER hast to be created first"
+        return 42
+    fi
+    export SERVICE_HOME
+
     git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \
-	      "$SEARX_GIT_BRANCH" "$SERVICE_USER"
+              "$SEARX_GIT_BRANCH" "$SERVICE_USER"
 
     pushd "${SEARX_SRC}" > /dev/null
     tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
 cd "${SEARX_SRC}"
 git config user.email "$ADMIN_EMAIL"
 git config user.name "$ADMIN_NAME"
-git checkout "$SEARX_GIT_BRANCH"
+git config --list
 EOF
     popd > /dev/null
 }
 
-create_venv(){
+remove_searx() {
-    rst_title "Create virtualenv (python)" section
+    rst_title "Drop searx sources" section
+    if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then
+        rm -rf "$SEARX_SRC"
+    else
+        rst_para "Leave searx sources unchanged."
+    fi
+}
 
-    rst_para "Create venv in ${SEARX_VENV} and install needed python packages."
+create_pyenv(){
+    rst_title "Create virtualenv (python)" section
     echo
+    if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then
+        err_msg "to create pyenv for searx, searx has to be cloned first"
+        return 42
+    fi
+    info_msg "create pyenv in ${SEARX_PYENV}"
     tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
-rm -rf "${SEARX_VENV}"
+rm -rf "${SEARX_PYENV}"
-python3 -m venv "${SEARX_VENV}"
+python3 -m venv "${SEARX_PYENV}"
-. ${SEARX_VENV}/bin/activate
+grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \
-${SEARX_SRC}/manage.sh update_packages
+  || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile
+EOF
+    info_msg "inspect python's virtual environment"
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+command -v python && python --version
 EOF
+    wait_key
+    info_msg "install needed python packages"
     tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
-grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \
+${SEARX_SRC}/manage.sh update_packages
-  || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile
 EOF
+}
 
+remove_pyenv(){
+    rst_title "Remove virtualenv (python)" section
+    if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then
+        return
+    fi
+    info_msg "remove pyenv activation from ~/.profile"
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.##
+mv ~/.profile.## ~/.profile
+EOF
+    rm -rf "${SEARX_PYENV}"
 }
 
 configure_searx(){
@@ -283,7 +330,7 @@ EOF
 }
 
 test_local_searx(){
-    rstHeading "Testing searx instance localy" section
+    rst_title "Testing searx instance localy" section
     echo
     tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
 cd ${SEARX_SRC}
@@ -293,7 +340,6 @@ sleep 1
 curl --location --verbose --head --insecure http://127.0.0.1:8888/
 sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS"
 EOF
-    waitKEY
 }
 
 install_searx_uwsgi() {

utils/templates/etc/uwsgi/apps-available/searx.ini

@@ -0,0 +1,62 @@
+[uwsgi]
+
+# uWSGI core
+# ----------
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
+
+# Who will run the code
+uid = ${SERVICE_USER}
+gid = ${SERVICE_GROUP}
+
+# chdir to specified directory before apps loading
+chdir = ${SEARX_SRC}
+
+# disable logging for privacy
+disable-logging = true
+
+# The right granted on the created socket
+chmod-socket = 666
+
+# Plugin to use and interpretor config
+single-interpreter = true
+
+# enable master process
+master = true
+
+# load apps in each worker instead of the master
+lazy-apps = true
+
+# load uWSGI plugins
+plugin = python3
+
+# By default the Python plugin does not initialize the GIL.  This means your
+# app-generated threads will not run.  If you need threads, remember to enable
+# them with enable-threads.  Running uWSGI in multithreading mode (with the
+# threads options) will automatically enable threading support. This *strange*
+# default behaviour is for performance reasons.
+enable-threads = true
+
+
+# plugin: python
+# --------------
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
+
+# load a WSGI module
+module = searx.webapp
+
+# set PYTHONHOME/virtualenv
+virtualenv = ${SEARX_PYENV}
+
+# add directory (or glob) to pythonpath
+pythonpath = ${SERVICE_HOME}
+
+
+# plugin http
+# -----------
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
+
+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
+http = ${SEARX_URL}