utils/filtron.sh: various fix from first installation test (WIP)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

utils/filtron.sh

@@ -11,6 +11,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
 FILTRON_ETC="/etc/filtron"
 
+FILTRON_RULES="$FILTRON_ETC/rules.json"
+FILTRON_API="127.0.0.1:4005"
+FILTRON_LISTEN="127.0.0.1:4004"
+FILTRON_TARGET="127.0.0.1:8888"
+
 SERVICE_NAME="filtron"
 SERVICE_USER="${SERVICE_NAME}"
 SERVICE_HOME="/home/${SERVICE_USER}"
@@ -23,6 +28,11 @@ GO_ENV="${SERVICE_HOME}/.go_env"
 GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"
 GO_TAR=$(basename "$GO_PKG_URL")
 
+CONFIG_FILES=(
+    "${FILTRON_RULES}"
+    "${SERVICE_SYSTEMD_UNIT}"
+)
+
 # ----------------------------------------------------------------------------
 usage(){
 # ----------------------------------------------------------------------------
@@ -37,10 +47,16 @@ usage:
   $(basename "$0") remove     [all]
   $(basename "$0") activate   [server]
   $(basename "$0") deactivate [server]
-
-shell        - start interactive shell with user ${SERVICE_USER}
-install user - add service user '$SERVICE_USER' at $SERVICE_HOME
-
+  $(basename "$0") show       [server]
+
+shell
+  start interactive shell from user ${SERVICE_USER}
+show server
+  show server status and log
+install / remove
+  all  - complete setup of filtron server
+install user
+  add service user '$SERVICE_USER' at $SERVICE_HOME
 EOF
     [ ! -z ${1+x} ] &&  echo -e "$1"
 }
@@ -58,6 +74,14 @@ main(){
 	    sudo_or_exit
 	    interactive_shell
 	    ;;
+        show)
+            case $2 in
+                server)
+		    sudo_or_exit
+		    show_server
+		    ;;
+                *) usage "$_usage"; exit 42;;
+            esac ;;
         install)
             sudo_or_exit
             case $2 in
@@ -91,21 +115,27 @@ main(){
 install_all() {
     rst_title "Install $SERVICE_NAME (service)"
     assert_user
+    wait_key
     install_go
+    wait_key
     install_filtron
+    wait_key
     install_server
+    wait_key
 }
 
 remove_all() {
     rst_title "De-Install $SERVICE_NAME (service)"
     remove_server
+    wait_key
     remove_user
-    rm -rf "$FILTRON_ETC"
+    rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
     wait_key
 }
 
 install_server() {
-    rst_title "Install System-D Unit ${SERVICE_NAME}.service ..." section
+    rst_title "Install System-D Unit ${SERVICE_NAME}.service" section
+    echo
     install_template ${SERVICE_SYSTEMD_UNIT} root root 644
     wait_key
     activate_server
@@ -116,12 +146,12 @@ remove_server() {
         return
     fi
     deactivate_server
-    rm "${SERVICE_SYSTEMD_UNIT}"
+    rm "${SERVICE_SYSTEMD_UNIT}"  2>&1 | prefix_stdout
 }
 
-
 activate_server () {
     rst_title "Activate $SERVICE_NAME (service)" section
+    echo
     tee_stderr <<EOF | bash 2>&1 | prefix_stdout
 systemctl enable $SERVICE_NAME.service
 systemctl restart $SERVICE_NAME.service
@@ -129,7 +159,6 @@ EOF
     tee_stderr <<EOF | bash 2>&1 | prefix_stdout
 systemctl status $SERVICE_NAME.service
 EOF
-    wait_key
 }
 
 deactivate_server () {
@@ -139,7 +168,6 @@ deactivate_server () {
 systemctl stop $SERVICE_NAME.service
 systemctl disable $SERVICE_NAME.service
 EOF
-    wait_key
 }
 
 assert_user() {
@@ -168,18 +196,18 @@ EOF
 remove_user() {
     rst_title "Drop $SERVICE_USER HOME" section
     if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then
-        userdel -r -f "$SERVICE_USER"
+        userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout
     else
         rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged."
     fi
 }
 
 interactive_shell(){
-    echo "// exit with STRG-D"
+    echo "// exit with CTRL-D"
     sudo -H -u ${SERVICE_USER} -i
 }
 
-_service_prefix="$SERVICE_USER@$(hostname) -->| "
+_service_prefix="  |$SERVICE_USER| "
 
 install_go(){
     rst_title "Install Go in user's HOME" section
@@ -199,14 +227,29 @@ EOF
 ! which go >/dev/null &&  echo "Go Installation not found in PATH!?!"
 which go >/dev/null &&  go version && echo "congratulations -- Go installation OK :)"
 EOF
-    wait_key
 }
 
 install_filtron() {
-    tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" | prefix_stdout "$_service_prefix"
-go get -v -u github.com/asciimoo/filtron 2>&1
+    rst_title "Install filtron in user's ~/go-apps" section
+    echo
+    tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_service_prefix"
+go get -v -u github.com/asciimoo/filtron
 EOF
-    install_template "$FILTRON_ETC/rules.json" root root 644
+    install_template --no-eval "$FILTRON_RULES" root root 644
+}
+
+show_server () {
+    rst_title "server status & log"
+    echo
+    systemctl status filtron.service
+    echo
+    read -s -n1 -t 5  -p "// use CTRL-C to stop monitoring the log"
+    echo
+    while true;  do
+	trap break 2
+	journalctl -f -u filtron
+    done
+    return 0
 }
 
 # ----------------------------------------------------------------------------

utils/lib.sh

@@ -22,6 +22,13 @@ if [[ -z "$SYSTEMD_UNITS" ]]; then
     SYSTEMD_UNITS="/lib/systemd/system"
 fi
 
+if [[ -z ${DIFF_CMD} ]]; then
+    DIFF_CMD="diff -u"
+    if command -v colordiff >/dev/null;  then
+        DIFF_CMD="colordiff -u"
+    fi
+fi
+
 sudo_or_exit() {
     # usage: sudo_or_exit
 
@@ -55,10 +62,10 @@ rst_para() {
     # usage:  RST_INDENT=1 rst_para "lorem ipsum ..."
     local prefix=''
     if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then
-	prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf "  "; done)"
-	echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
+        prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf "  "; done)"
+        echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
     else
-	echo -en "\n$*\n" | $FMT
+        echo -en "\n$*\n" | $FMT
     fi
 }
 
@@ -66,15 +73,23 @@ err_msg()  { echo -e "ERROR: $*" >&2; }
 warn_msg() { echo -e "WARN:  $*" >&2; }
 info_msg() { echo -e "INFO:  $*"; }
 
+clean_stdin() {
+    if [[ $(uname -s) != 'Darwin' ]]; then
+        while $(read -n1 -t 0.1); do : ; done
+    fi
+}
+
 wait_key(){
     # usage: waitKEY [<timeout in sec>]
 
+    clean_stdin
     local _t=$1
     [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
     [[ ! -z $_t ]] && _t="-t $_t"
     # shellcheck disable=SC2086
-    read -n1 $_t -p "** press any [KEY] to continue **"
+    read -s -n1 $_t -p "** press any [KEY] to continue **"
     echo
+    clean_stdin
 }
 
 ask_yn() {
@@ -100,6 +115,7 @@ ask_yn() {
     esac
     echo
     while true; do
+	clean_stdin
         printf "$1 ${choice} "
         # shellcheck disable=SC2086
         read -n1 $_t
@@ -117,6 +133,7 @@ ask_yn() {
         _t=""
         err_msg "invalid choice"
     done
+    clean_stdin
     return $exit_val
 }
 
@@ -144,7 +161,7 @@ tee_stderr () {
 prefix_stdout () {
     # usage: <cmd> | prefix_stdout [prefix]
 
-    local prefix="-->| "
+    local prefix="  | "
 
     if [[ ! -z $1 ]] ; then prefix="$1"; fi
 
@@ -223,6 +240,7 @@ choose_one() {
         fi
     done
     while true; do
+	clean_stdin
         printf "$1 [$default] "
 
         if (( 10 > $max )); then
@@ -242,6 +260,7 @@ choose_one() {
         err_msg "invalid choice"
     done
     echo
+    clean_stdin
     eval "$env_name"='${list[${REPLY}]}'
 }
 
@@ -288,31 +307,48 @@ install_template() {
 
     mkdir -p "$(dirname "${dst}")"
 
-    if [[ -f "${dst}" ]] ; then
-        info_msg "file ${dst} allready exists on this host"
+    if [[ ! -f "${dst}" ]]; then
+        info_msg "install: ${template_file}"
+        sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
+             "${template_file}" "${dst}" | prefix_stdout
+        return $?
+    fi
+
+    if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then
+        info_msg "file ${dst} allready installed"
+        return 0
+    fi
+
+    info_msg "file ${dst} allready exists on this host"
+
+    while true; do
         choose_one _reply "choose next step with file $dst" \
                    "replace file" \
-                   "leave file unchanged"
+                   "leave file unchanged" \
+                   "interactiv shell" \
+                   "diff files"
 
         case $_reply in
             "replace file")
                 info_msg "install: ${template_file}"
                 sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
                      "${template_file}" "${dst}" | prefix_stdout
+		break
                 ;;
             "leave file unchanged")
+                break
                 ;;
             "interactiv shell")
-                echo "// exit with STRG-D"
+                echo "// edit ${dst} to your needs"
+                echo "// exit with CTRL-D"
                 sudo -H -u "${owner}" -i
-            ;;
+                $DIFF_CMD "${dst}" "${template_file}"
+		if ask_yn "did you edit ${template_file} to your needs?"; then
+		    break
+		fi
+                ;;
+            "diff files")
+                $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
         esac
-
-    else
-        info_msg "install: ${template_file}"
-        sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
-             "${template_file}" "${dst}" | prefix_stdout
-    fi
-
+    done
 }
-

utils/templates/etc/filtron/rules.json

@@ -1,56 +1,98 @@
-[
+[{
+  "name":"search request",
+  "filters":[
+    "Param:q",
+    "Path=^(/|/search)$"
+  ],
+  "interval":60,
+  "limit":15,
+  "subrules":[
     {
-        "name": "api limit",
-        "interval": 60,
-        "limit": 1000,
-        "filters": ["Path=^/api"],
-        "aggregations": ["Path"],
-        "actions": [
-            {"name": "block"}
-        ],
-        "subrules": [
-            {
-                "name": "drop put",
-                "interval": 60,
-                "limit": 100,
-                "filters": ["Method=PUT"],
-                "aggregations": ["Header:X-Forwarded-For"],
-                "actions": [
-                    {"name": "shell",
-                     "params": {"cmd": "iptables -A INPUT -s %v -j DROP", "args": ["Header:X-Forwarded-For"]}}
-                ]
-            }
-        ]
+      "name":"roboagent limit",
+      "interval":60,
+      "limit":15,
+      "filters":[
+        "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
+      ],
+      "actions":[
+	{"name": "log"},
+	{
+          "name":"block",
+          "params":{
+            "message":"Rate limit exceeded"
+          }
+        }
+      ]
     },
     {
-        "name": "log'n'block rss",
-        "interval": 300,
-        "limit": 2500,
-        "filters": ["Path=^/$", "GET:format=rss"],
-        "actions": [
-            {"name": "log"},
-            {"name": "block"}
-        ]
+      "name":"botlimit",
+      "limit":0,
+      "stop":true,
+      "filters":[
+        "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
+      ],
+      "actions":[
+	{"name": "log"},
+        {
+          "name":"block",
+          "params":{
+            "message":"Rate limit exceeded"
+          }
+        }
+      ]
     },
     {
-        "name": "log rule",
-        "filters": ["Path=/"],
-        "actions": [ {"name": "log"} ],
-        "subrules": [
-            {
-                "name": "block missing accept-language",
-                "filters": ["!Header:Accept-Language"],
-                "actions": [
-                    {"name": "block"}
-                ]
-            },
-            {
-                "name": "block curl",
-                "filters": ["Header:User-Agent=[Cc]url"],
-                "actions": [
-                    {"name": "block"}
-                ]
-            }
-        ]
+      "name":"IP limit",
+      "interval":60,
+      "limit":15,
+      "stop":true,
+      "aggregations":[
+        "Header:X-Forwarded-For"
+      ],
+      "actions":[
+	{"name": "log"},
+        {
+          "name":"block",
+          "params":{
+            "message":"Rate limit exceeded"
+          }
+        }
+      ]
+    },
+    {
+      "name":"rss/json limit",
+      "interval":60,
+      "limit":15,
+      "stop":true,
+      "filters":[
+        "Param:format=(csv|json|rss)"
+      ],
+      "actions":[
+	{"name": "log"},
+        {
+          "name":"block",
+          "params":{
+            "message":"Rate limit exceeded"
+          }
+        }
+      ]
+      },
+    {
+      "name":"useragent limit",
+      "interval":60,
+      "limit":15,
+      "aggregations":[
+        "Header:User-Agent"
+      ],
+      "actions":[
+	{"name": "log"},
+        {
+          "name":"block",
+          "params":{
+            "message":"Rate limit exceeded"
+          }
+        }
+      ]
     }
-]
+  ]
+}]

utils/templates/lib/systemd/system/filtron.service

@@ -10,7 +10,7 @@ Type=simple
 User=${SERVICE_USER}
 Group=${SERVICE_GROUP}
 WorkingDirectory=${SERVICE_HOME}
-ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -rules ${FILTRON_RULES}
+ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}'
 
 Restart=always
 Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME}