/etc/filtron/rules.json: normalize rules from docs & tooling box

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

docs/admin/filtron.rst

@@ -39,6 +39,10 @@ an installation and the maintenance of, use our script :ref:`filtron.sh`.
 Sample configuration of filtron
 ===============================
 
+.. sidebar:: Tooling box
+
+   - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>`
+
 An example configuration can be find below. This configuration limits the access
 of:
 
@@ -50,100 +54,100 @@ of:
 
 .. code:: json
 
-   [{
-      "name":"search request",
-      "filters":[
-         "Param:q",
-         "Path=^(/|/search)$"
-      ],
-      "interval":"<time-interval-in-sec (int)>",
-      "limit":"<max-request-number-in-interval (int)>",
-      "subrules":[
-         {
-            "name":"roboagent limit",
-            "interval":"<time-interval-in-sec (int)>",
-            "limit":"<max-request-number-in-interval (int)>",
-            "filters":[
-               "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
-            ],
-            "actions":[
-               {
-                  "name":"block",
-                  "params":{
-                     "message":"Rate limit exceeded"
-                  }
-               }
-            ]
-         },
-         {
-            "name":"botlimit",
-            "limit":0,
-            "stop":true,
-            "filters":[
-               "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
-            ],
-            "actions":[
-               {
-                  "name":"block",
-                  "params":{
-                     "message":"Rate limit exceeded"
-                  }
-               }
-            ]
-         },
-         {
-            "name":"IP limit",
-            "interval":"<time-interval-in-sec (int)>",
-            "limit":"<max-request-number-in-interval (int)>",
-            "stop":true,
-            "aggregations":[
-               "Header:X-Forwarded-For"
-            ],
-            "actions":[
-               {
-                  "name":"block",
-                  "params":{
-                     "message":"Rate limit exceeded"
-                  }
-               }
-            ]
-         },
-         {
-            "name":"rss/json limit",
-            "interval":"<time-interval-in-sec (int)>",
-            "limit":"<max-request-number-in-interval (int)>",
-            "stop":true,
-            "filters":[
-               "Param:format=(csv|json|rss)"
-            ],
-            "actions":[
-               {
-                  "name":"block",
-                  "params":{
-                     "message":"Rate limit exceeded"
-                  }
-               }
-            ]
-         },
-         {
-            "name":"useragent limit",
-            "interval":"<time-interval-in-sec (int)>",
-            "limit":"<max-request-number-in-interval (int)>",
-            "aggregations":[
-               "Header:User-Agent"
-            ],
-            "actions":[
-               {
-                  "name":"block",
-                  "params":{
-                     "message":"Rate limit exceeded"
-                  }
-               }
-            ]
-         }
-      ]
-   }]
-
+   [
+     { "name": "search request",
+       "filters": [
+	 "Param:q",
+	 "Path=^(/|/search)$"
+       ],
+       "interval": "<time-interval-in-sec (int)>",
+       "limit": "<max-request-number-in-interval (int)>",
+       "subrules": [
+	 {
+	   "name": "roboagent limit",
+	   "interval": "<time-interval-in-sec (int)>",
+	   "limit": "<max-request-number-in-interval (int)>",
+	   "filters": [
+	     "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
+	   ],
+	   "actions": [
+	     { "name": "log"},
+	     { "name": "block",
+	       "params": {
+		 "message": "Rate limit exceeded"
+	       }
+	     }
+	   ]
+	 },
+	 {
+	   "name": "botlimit",
+	   "limit": 0,
+	   "stop": true,
+	   "filters": [
+	     "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
+	   ],
+	   "actions": [
+	     { "name": "log"},
+	     { "name": "block",
+	       "params": {
+		 "message": "Rate limit exceeded"
+	       }
+	     }
+	   ]
+	 },
+	 {
+	   "name": "IP limit",
+	   "interval": "<time-interval-in-sec (int)>",
+	   "limit": "<max-request-number-in-interval (int)>",
+	   "stop": true,
+	   "aggregations": [
+	     "Header:X-Forwarded-For"
+	   ],
+	   "actions": [
+	     { "name": "log"},
+	     { "name": "block",
+	       "params": {
+		 "message": "Rate limit exceeded"
+	       }
+	     }
+	   ]
+	 },
+	 {
+	   "name": "rss/json limit",
+	   "interval": "<time-interval-in-sec (int)>",
+	   "limit": "<max-request-number-in-interval (int)>",
+	   "stop": true,
+	   "filters": [
+	     "Param:format=(csv|json|rss)"
+	   ],
+	   "actions": [
+	     { "name": "log"},
+	     { "name": "block",
+	       "params": {
+		 "message": "Rate limit exceeded"
+	       }
+	     }
+	   ]
+	 },
+	 {
+	   "name": "useragent limit",
+	   "interval": "<time-interval-in-sec (int)>",
+	   "limit": "<max-request-number-in-interval (int)>",
+	   "aggregations": [
+	     "Header:User-Agent"
+	   ],
+	   "actions": [
+	     { "name": "log"},
+	     { "name": "block",
+	       "params": {
+		 "message": "Rate limit exceeded"
+	       }
+	     }
+	   ]
+	 }
+       ]
+     }
+   ]
 
 
 Route request through filtron

utils/filtron.sh

@@ -94,8 +94,9 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"
   FILTRON_API    : ${FILTRON_API}
   FILTRON_LISTEN : ${FILTRON_LISTEN}
   FILTRON_TARGET : ${FILTRON_TARGET}
+
 EOF
-    [ ! -z ${1+x} ] &&  echo -e "$1"
+    [ ! -z ${1+x} ] &&  err_msg "$1"
 }
 
 main() {
@@ -105,7 +106,7 @@ main() {
         dpkg apt-get install git wget curl \
         || exit
 
-    local _usage="ERROR: unknown or missing $1 command $2"
+    local _usage="unknown or missing $1 command $2"
 
     case $1 in
         --source-only)  ;;
@@ -170,7 +171,7 @@ main() {
                 *) usage "$_usage"; exit 42;;
             esac ;;
 
-        *) usage "ERROR: unknown or missing command $1"; exit 42;;
+        *) usage "unknown or missing command $1"; exit 42;;
     esac
 }
 

utils/morty.sh

@@ -84,7 +84,7 @@ ${DOT_CONFIG#"$REPO_ROOT/"} file::
   SERVICE_USER :   ${SERVICE_USER}
 EOF
     info_searx
-    [ ! -z ${1+x} ] &&  echo -e "$1"
+    [ ! -z ${1+x} ] &&  err_msg "$1"
 }
 
 info_searx() {

utils/searx.sh

@@ -110,7 +110,7 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"
   SERVICE_USER        : ${SERVICE_USER}
 
 EOF
-    [ ! -z ${1+x} ] &&  echo -e "$1"
+    [ ! -z ${1+x} ] &&  err_msg "$1"
 }
 
 main() {
@@ -120,7 +120,7 @@ main() {
         dpkg systemctl apt-get install git wget curl \
         || exit
 
-    local _usage="ERROR: unknown or missing $1 command $2"
+    local _usage="unknown or missing $1 command $2"
 
     case $1 in
         --source-only)  ;;
@@ -190,7 +190,7 @@ main() {
                 *) usage "$_usage"; exit 42;;
             esac ;;
 
-        *) usage "ERROR: unknown or missing command $1"; exit 42;;
+        *) usage "unknown or missing command $1"; exit 42;;
     esac
 }
 

utils/templates/etc/filtron/rules.json

@@ -1,107 +1,105 @@
-[{
-  "name":"search request",
-  "filters":[
-    "Param:q",
-    "Path=^(/|/search)$"
-  ],
-  "interval":120,
-  "limit":19,
-  "actions":[
-    {
-      "name":"block",
-      "params":{
-        "message":"common rate limit exceeded"
-      }
-    }
-  ],
-  "subrules":[
-    {
-      "name":"roboagent limit",
-      "interval":60,
-      "limit":3,
-      "filters":[
-        "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby)"
-      ],
-      "actions":[
-        {"name":"log"},
-        {
-          "name":"block",
-          "params":{
-            "message":"rate limit exceeded"
+[
+  { "name": "suspiciously frequent IP",
+    "filters": [],
+    "interval": 600,
+    "limit": 30,
+    "aggregations": [
+      "Header:X-Forwarded-For"
+    ],
+    "actions":[
+      {"name":"log"}
+    ]
+  },
+  { "name": "search request",
+    "filters": [
+      "Param:q",
+      "Path=^(/|/search)$"
+    ],
+    "interval": 61,
+    "limit": 999,
+    "subrules": [
+      {
+        "name": "roboagent limit",
+        "interval": 61,
+        "limit": 1,
+        "filters": [
+          "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
+        ],
+        "actions": [
+          { "name": "log"},
+          { "name": "block",
+            "params": {
+              "message": "Rate limit exceeded"
+            }
           }
-        }
-      ]
-    },
-    {
-      "name":"botlimit",
-      "interval":60,
-      "limit":0,
-      "stop":true,
-      "filters":[
-        "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
-      ],
-      "actions":[
-        {"name":"log"},
-        {
-          "name":"block",
-          "params":{
-            "message":"rate limit exceeded"
+        ]
+      },
+      {
+        "name": "botlimit",
+        "limit": 0,
+        "stop": true,
+        "filters": [
+          "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
+        ],
+        "actions": [
+          { "name": "log"},
+          { "name": "block",
+            "params": {
+              "message": "Rate limit exceeded"
+            }
           }
-        }
-      ]
-    },
-    {
-      "name":"IP limit",
-      "interval":60,
-      "limit":13,
-      "stop":true,
-      "aggregations":[
-        "Header:X-Forwarded-For"
-      ],
-      "actions":[
-        {"name":"log"},
-        {
-          "name":"block",
-          "params":{
-            "message":"rate limit exceeded"
+        ]
+      },
+      {
+        "name": "IP limit",
+        "interval": 61,
+        "limit": 9,
+        "stop": true,
+        "aggregations": [
+          "Header:X-Forwarded-For"
+        ],
+        "actions": [
+          { "name": "log"},
+          { "name": "block",
+            "params": {
+              "message": "Rate limit exceeded"
+            }
           }
-        }
-      ]
-    },
-    {
-      "name":"rss/json limit",
-      "interval":60,
-      "limit":13,
-      "stop":true,
-      "filters":[
-        "Param:format=(csv|json|rss)"
-      ],
-      "actions":[
-        {"name":"log"},
-        {
-          "name":"block",
-          "params":{
-            "message":"rate limit exceeded"
+        ]
+      },
+      {
+        "name": "rss/json limit",
+        "interval": 121,
+        "limit": 2,
+        "stop": true,
+        "filters": [
+          "Param:format=(csv|json|rss)"
+        ],
+        "actions": [
+          { "name": "log"},
+          { "name": "block",
+            "params": {
+              "message": "Rate limit exceeded"
+            }
           }
-        }
-      ]
-    },
-    {
-      "name":"useragent limit",
-      "interval":60,
-      "limit":13,
-      "aggregations":[
-        "Header:User-Agent"
-      ],
-      "actions":[
-        {"name":"log"},
-        {
-          "name":"block",
-          "params":{
-            "message":"rate limit exceeded"
+        ]
+      },
+      {
+        "name": "useragent limit",
+        "interval": 61,
+        "limit": 199,
+        "aggregations": [
+          "Header:User-Agent"
+        ],
+        "actions": [
+          { "name": "log"},
+          { "name": "block",
+            "params": {
+              "message": "Rate limit exceeded"
+            }
           }
-        }
-      ]
-    }
-  ]
-}]
+        ]
+      }
+    ]
+  }
+]