tooling box: added nginx + polished bash scripts and environment

- add installation method for nginx sites, morty and filtron
- clean up PUBLIC_URL environment in and outside of containers
- clean up comand lines
- handle uWSGI quirks on fedora (emperor mode)
- handle Python quirks on debian (there is no 'python' command anymore)
- lib.sh: add die and die_caller functions
- lxc_suite_install_info is now a function
- lint: shellcheck

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.config.sh

@@ -18,7 +18,8 @@
 PUBLIC_URL="${SEARX_URL}"
 
 if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then
-    PUBLIC_URL=
+    # hint: Linux containers do not have DNS entries, lets use IPs
+    PUBLIC_URL="http://$(primary_ip)/searx"
 fi
 
 # searx.sh

utils/filtron.sh

@@ -225,6 +225,11 @@ install_all() {
         if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
             install_apache_site
         fi
+    elif nginx_is_installed; then
+        info_msg "nginx is installed on this host."
+        if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
+            install_nginx_site
+        fi
     fi
     if ask_yn "Do you want to inspect the installation?" Ny; then
         inspect_service
@@ -315,8 +320,6 @@ sourced ${DOT_CONFIG#"$REPO_ROOT/"} :
 
 EOF
 
-    apache_is_installed && info_msg "Apache is installed."
-
     if service_account_is_available "$SERVICE_USER"; then
         info_msg "service account $SERVICE_USER available."
     else
@@ -445,7 +448,7 @@ install_apache_site() {
     rst_para "\
 This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
 
-    ! apache_is_installed && err_msg "Apache is not installed."
+    ! apache_is_installed && info_msg "Apache is not installed."
 
     if ! ask_yn "Do you really want to continue?" Yn; then
         return
@@ -487,7 +490,7 @@ install_nginx_site() {
     rst_para "\
 This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
 
-    ! nginx_is_installed && err_msg "nginx is not installed."
+    ! nginx_is_installed && info_msg "nginx is not installed."
 
     if ! ask_yn "Do you really want to continue?" Yn; then
         return
@@ -497,7 +500,9 @@ This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE}
 
     "${REPO_ROOT}/utils/searx.sh" install uwsgi
 
+    # shellcheck disable=SC2034
     SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
+    # shellcheck disable=SC2034
     SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
     nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
 

utils/lib.sh

@@ -482,7 +482,7 @@ service_is_available() {
     # usage:  service_is_available <URL>
 
     [[ -z $1 ]] && die_caller 42 "missing argument <URL>"
-
+    local URL="$1"
     http_code=$(curl -H 'Cache-Control: no-cache' \
          --silent -o /dev/null --head --write-out '%{http_code}' --insecure \
          "${URL}")
@@ -969,6 +969,7 @@ uWSGI_distro_setup() {
             # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067
             uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
             uWSGI_APPS_ENABLED="${uWSGI_SETUP}/apps-enabled"
+            uWSGI_PACKAGES="uwsgi"
             ;;
         arch-*)
             # systemd --> /usr/lib/systemd/system/uwsgi@.service
@@ -977,6 +978,7 @@ uWSGI_distro_setup() {
             # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
             uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-archlinux"
             uWSGI_APPS_ENABLED="${uWSGI_SETUP}"
+            uWSGI_PACKAGES="uwsgi"
             ;;
         fedora-*)
             # systemd --> /usr/lib/systemd/system/uwsgi.service
@@ -984,6 +986,7 @@ uWSGI_distro_setup() {
             # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
             uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
             uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d"
+            uWSGI_PACKAGES="uwsgi"
             uWSGI_USER="uwsgi"
             uWSGI_GROUP="uwsgi"
             ;;
@@ -995,18 +998,26 @@ esac
 
 uWSGI_distro_setup
 
+install_uwsgi(){
+    info_msg "installing uwsgi ..."
+    pkg_install "$uWSGI_PACKAGES"
+    case $DIST_ID-$DIST_VERS in
+        fedora-*)
+            # enable & start should be called once at uWSGI installation time
+            systemctl enable uwsgi
+            systemctl restart uwsgi
+            ;;
+    esac
+}
+
 uWSGI_restart() {
 
     # usage:  uWSGI_restart() <myapp.ini>
 
     local CONF="$1"
-    if [[ -z $CONF ]]; then
-        err_msg "uWSGI_restart: missing arguments"
-        return 42
-    fi
 
+    [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
     info_msg "restart uWSGI service"
-
     case $DIST_ID-$DIST_VERS in
         ubuntu-*|debian-*)
             # the 'service' method seems broken in that way, that it (re-)starts
@@ -1025,6 +1036,9 @@ uWSGI_restart() {
             # in emperor mode, just touch the file to restart
             if uWSGI_app_enabled "${CONF}"; then
                 touch "${uWSGI_APPS_ENABLED}/${CONF}"
+                # it seems, there is a polling time in between touch and restart
+                # of the service.
+                sleep 3
             else
                 info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
             fi
@@ -1040,11 +1054,9 @@ uWSGI_prepare_app() {
 
     # usage:  uWSGI_prepare_app <myapp.ini>
 
+    [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>"
+
     local APP="${1%.*}"
-    if [[ -z $APP ]]; then
-        err_msg "uWSGI_prepare_app: missing arguments"
-        return 42
-    fi
 
     case $DIST_ID-$DIST_VERS in
         fedora-*)
@@ -1065,10 +1077,8 @@ uWSGI_prepare_app() {
 uWSGI_app_available() {
     # usage:  uWSGI_app_available <myapp.ini>
     local CONF="$1"
-    if [[ -z $CONF ]]; then
-        err_msg "uWSGI_app_available: missing arguments"
-        return 42
-    fi
+
+    [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
     [[ -f "${uWSGI_APPS_AVAILABLE}/${CONF}" ]]
 }
 
@@ -1101,6 +1111,8 @@ uWSGI_remove_app() {
     # usage:  uWSGI_remove_app <myapp.ini>
 
     local CONF="$1"
+
+    [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
     info_msg "remove uWSGI app: ${CONF}"
     uWSGI_disable_app "${CONF}"
     uWSGI_restart "${CONF}"
@@ -1110,12 +1122,10 @@ uWSGI_remove_app() {
 uWSGI_app_enabled() {
     # usage:  uWSGI_app_enabled <myapp.ini>
 
-    local CONF="$1"
     local exit_val=0
-    if [[ -z $CONF ]]; then
-        err_msg "uWSGI_app_enabled: missing arguments"
-        return 42
-    fi
+    local CONF="$1"
+
+    [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
     case $DIST_ID-$DIST_VERS in
         ubuntu-*|debian-*)
             [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]
@@ -1145,11 +1155,7 @@ uWSGI_enable_app() {
 
     local CONF="$1"
 
-    if [[ -z $CONF ]]; then
-        err_msg "uWSGI_enable_app: missing arguments"
-        return 42
-    fi
-
+    [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
     case $DIST_ID-$DIST_VERS in
         ubuntu-*|debian-*)
             mkdir -p "${uWSGI_APPS_ENABLED}"
@@ -1183,11 +1189,8 @@ uWSGI_disable_app() {
     # usage:   uWSGI_disable_app <myapp.ini>
 
     local CONF="$1"
-    if [[ -z $CONF ]]; then
-        err_msg "uWSGI_disable_app: missing arguments"
-        return 42
-    fi
 
+    [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
     case $DIST_ID-$DIST_VERS in
         ubuntu-*|debian-*)
             service uwsgi stop "${CONF%.*}"

utils/lxc-searx.env

@@ -32,13 +32,36 @@ lxc_set_suite_env() {
         # rolling releases see https://www.archlinux.org/releng/releases/
         "$LINUXCONTAINERS_ORG_NAME:archlinux"     "archlinux"
     )
-    export FILTRON_API="0.0.0.0:4005"
-    export FILTRON_LISTEN="0.0.0.0:4004"
-    export MORTY_LISTEN="0.0.0.0:3000"
+
+    PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
+    if in_container; then
+        # container hostnames do not have a DNS entry: use primary IP!
+        PUBLIC_URL="http://$(primary_ip)/searx"
+
+        # make GUEST's services public to the HOST
+        FILTRON_API="0.0.0.0:4005"
+        FILTRON_LISTEN="0.0.0.0:4004"
+        MORTY_LISTEN="0.0.0.0:3000"
+
+        # export LXC specific environment
+        export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN
+    fi
 }
 
-# shellcheck disable=SC2034
-LXC_SUITE_INSTALL_INFO="suite includes searx, morty & filtron"
+lxc_suite_install_info() {
+    (
+        lxc_set_suite_env
+        cat <<EOF
+LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL}
+  suite includes searx, morty & filtron
+suite images:
+$(echo "  ${LOCAL_IMAGES[*]}" | $FMT)
+suite containers:
+$(echo "  ${CONTAINERS[*]}" | $FMT)
+EOF
+    )
+    }
+
 lxc_suite_install() {
     (
         lxc_set_suite_env
@@ -62,8 +85,9 @@ lxc_suite_info() {
                 info_msg "(${ip%|*}) IPv6:       http://[${ip#*|}]"
             else
                 # IPv4:
-                info_msg "(${ip%|*}) filtron:    http://${ip#*|}:4004/"
-                info_msg "(${ip%|*}) morty:      http://${ip#*|}:3000/"
+                # shellcheck disable=SC2034,SC2031
+                info_msg "(${ip%|*}) filtron:    http://${ip#*|}:4004/ $PUBLIC_URL"
+                info_msg "(${ip%|*}) morty:      http://${ip#*|}:3000/ $PUBLIC_URL_MORTY"
                 info_msg "(${ip%|*}) docs-live:  http://${ip#*|}:8080/"
             fi
         done

utils/lxc.sh

@@ -112,14 +112,7 @@ EOF
 }
 
 usage_containers() {
-    cat <<EOF
-LXC suite: ${LXC_SUITE_NAME}
-$(echo "  ${LXC_SUITE_INSTALL_INFO}" | $FMT)
-suite images:
-$(echo "  ${LOCAL_IMAGES[*]}" | $FMT)
-suite containers:
-$(echo "  ${CONTAINERS[*]}" | $FMT)
-EOF
+    lxc_suite_install_info
     [ -n "${1+x}" ] &&  err_msg "$1"
 }
 
@@ -172,7 +165,7 @@ main() {
                 ''|--|containers) remove_containers ;;
                 images) lxc_delete_images_localy ;;
                 ${LXC_HOST_PREFIX}-*)
-                    ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42
+                    ! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0
                     if ask_yn "Do you really want to delete container $2"; then
                         lxc_delete_container "$2"
                     fi
@@ -201,7 +194,7 @@ main() {
                             lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
                                 | prefix_stdout "[${_BBlue}$3${_creset}]  "
                         ;;
-                        *|--) show_suite;;
+                        *) show_suite;;
                     esac
                     ;;
                 images) show_images ;;
@@ -211,7 +204,7 @@ main() {
                             ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
                             lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
                         ;;
-                        *|--)
+                        *)
                             rst_title "container configurations"
                             echo
                             lxc list "$LXC_HOST_PREFIX-"
@@ -226,7 +219,7 @@ main() {
                             ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
                             lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
                             ;;
-                        *|--)
+                        *)
                             rst_title "container info"
                             echo
                             lxc_cmd info
@@ -350,7 +343,9 @@ remove_containers() {
     echo
     lxc list "$LXC_HOST_PREFIX-"
     echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n  ${CONTAINERS[*]}\\n" | $FMT
-    if ask_yn "Do you really want to delete these containers"; then
+    local default=Ny
+    [[ $FORCE_TIMEOUT = 0 ]] && default=Yn
+    if ask_yn "Do you really want to delete these containers" $default; then
         for i in "${CONTAINERS[@]}"; do
             lxc_delete_container "$i"
         done

utils/morty.sh

@@ -7,6 +7,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 # shellcheck source=utils/brand.env
 source "${REPO_ROOT}/utils/brand.env"
 source_dot_config
+SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
 source "${REPO_ROOT}/utils/lxc-searx.env"
 in_container && lxc_set_suite_env
 
@@ -17,12 +18,7 @@ in_container && lxc_set_suite_env
 MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
 PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}"
 
-SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
-PUBLIC_URL_MORTY="$(echo "$SEARX_URL" |  sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
-if in_container; then
-    # container hostnames do not have a DNS entry, use primary IP
-    PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")"
-fi
+PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$SEARX_URL" |  sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}"
 
 # shellcheck disable=SC2034
 MORTY_TIMEOUT=5
@@ -229,6 +225,11 @@ install_all() {
         if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
             install_apache_site
         fi
+    elif nginx_is_installed; then
+        info_msg "nginx is installed on this host."
+        if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
+            install_nginx_site
+        fi
     fi
     info_searx
     if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then
@@ -462,7 +463,9 @@ This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})"
 
     "${REPO_ROOT}/utils/searx.sh" install uwsgi
 
+    # shellcheck disable=SC2034
     SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
+    # shellcheck disable=SC2034
     SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)
     nginx_install_app "${NGINX_MORTY_SITE}"
 

utils/searx.sh

@@ -81,6 +81,11 @@ case $DIST_ID-$DIST_VERS in
         BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
         APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi"
         ;;
+    ubuntu-20.04)
+        # https://askubuntu.com/a/1224710
+        SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3"
+        BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
+        ;;
     ubuntu-*|debian-*)
         SEARX_PACKAGES="${SEARX_PACKAGES_debian}"
         BUILD_PACKAGES="${BUILD_PACKAGES_debian}"
@@ -206,7 +211,12 @@ main() {
                 pyenv) create_pyenv ;;
                 searx-src) clone_searx ;;
                 settings) install_settings ;;
-                uwsgi) install_searx_uwsgi;;
+                uwsgi)
+                    install_searx_uwsgi
+                    if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then
+                        err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!"
+                    fi
+                    ;;
                 packages)
                     pkg_install "$SEARX_PACKAGES"
                     ;;
@@ -272,11 +282,6 @@ install_all() {
     rst_title "Install $SEARX_INSTANCE_NAME (service)"
     pkg_install "$SEARX_PACKAGES"
     wait_key
-    case $DIST_ID-$DIST_VERS in
-        fedora-*)
-            systemctl enable uwsgi
-            ;;
-    esac
     assert_user
     wait_key
     clone_searx
@@ -514,6 +519,7 @@ EOF
 install_searx_uwsgi() {
     rst_title "Install searx's uWSGI app (searx.ini)" section
     echo
+    install_uwsgi
     uWSGI_install_app "$SEARX_UWSGI_APP"
 }
 
@@ -575,7 +581,10 @@ EOF
 }
 
 set_result_proxy() {
-    info_msg "try to set result proxy ..."
+
+    # usage: set_result_proxy <URL> [<key>]
+
+    info_msg "try to set result proxy: $1"
     cp "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_PATH}.bak"
     _set_result_proxy "$1" "$2" > "${SEARX_SETTINGS_PATH}"
 }