From 2f40f61f83afb34f0c4b95019a53050a504ce43a Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 4 Feb 2020 17:59:58 +0100 Subject: [PATCH] /etc/filtron/rules.json: normalize rules from docs & tooling box Signed-off-by: Markus Heiser --- docs/admin/filtron.rst | 192 +++++++++++----------- utils/filtron.sh | 7 +- utils/morty.sh | 2 +- utils/searx.sh | 6 +- utils/templates/etc/filtron/rules.json | 210 ++++++++++++------------- 5 files changed, 210 insertions(+), 207 deletions(-) diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 205269973..2bc663411 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -39,6 +39,10 @@ an installation and the maintenance of, use our script :ref:`filtron.sh`. Sample configuration of filtron =============================== +.. sidebar:: Tooling box + + - :origin:`/etc/filtron/rules.json ` + An example configuration can be find below. This configuration limits the access of: @@ -50,100 +54,100 @@ of: .. code:: json - [{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":"", - "limit":"", - "subrules":[ - { - "name":"roboagent limit", - "interval":"", - "limit":"", - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"botlimit", - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"IP limit", - "interval":"", - "limit":"", - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"rss/json limit", - "interval":"", - "limit":"", - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"useragent limit", - "interval":"", - "limit":"", - "aggregations":[ - "Header:User-Agent" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - } - ] - }] - + [ + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": "", + "limit": "", + "subrules": [ + { + "name": "roboagent limit", + "interval": "", + "limit": "", + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "IP limit", + "interval": "", + "limit": "", + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "interval": "", + "limit": "", + "stop": true, + "filters": [ + "Param:format=(csv|json|rss)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": "", + "limit": "", + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } + ] Route request through filtron diff --git a/utils/filtron.sh b/utils/filtron.sh index 3c545ff14..c1e360067 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -94,8 +94,9 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/" FILTRON_API : ${FILTRON_API} FILTRON_LISTEN : ${FILTRON_LISTEN} FILTRON_TARGET : ${FILTRON_TARGET} + EOF - [ ! -z ${1+x} ] && echo -e "$1" + [ ! -z ${1+x} ] && err_msg "$1" } main() { @@ -105,7 +106,7 @@ main() { dpkg apt-get install git wget curl \ || exit - local _usage="ERROR: unknown or missing $1 command $2" + local _usage="unknown or missing $1 command $2" case $1 in --source-only) ;; @@ -170,7 +171,7 @@ main() { *) usage "$_usage"; exit 42;; esac ;; - *) usage "ERROR: unknown or missing command $1"; exit 42;; + *) usage "unknown or missing command $1"; exit 42;; esac } diff --git a/utils/morty.sh b/utils/morty.sh index ba4e7dd2e..6c63351cf 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -84,7 +84,7 @@ ${DOT_CONFIG#"$REPO_ROOT/"} file:: SERVICE_USER : ${SERVICE_USER} EOF info_searx - [ ! -z ${1+x} ] && echo -e "$1" + [ ! -z ${1+x} ] && err_msg "$1" } info_searx() { diff --git a/utils/searx.sh b/utils/searx.sh index 707009a11..7609a4acf 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -110,7 +110,7 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/" SERVICE_USER : ${SERVICE_USER} EOF - [ ! -z ${1+x} ] && echo -e "$1" + [ ! -z ${1+x} ] && err_msg "$1" } main() { @@ -120,7 +120,7 @@ main() { dpkg systemctl apt-get install git wget curl \ || exit - local _usage="ERROR: unknown or missing $1 command $2" + local _usage="unknown or missing $1 command $2" case $1 in --source-only) ;; @@ -190,7 +190,7 @@ main() { *) usage "$_usage"; exit 42;; esac ;; - *) usage "ERROR: unknown or missing command $1"; exit 42;; + *) usage "unknown or missing command $1"; exit 42;; esac } diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json index f333306e1..1c7005ae5 100644 --- a/utils/templates/etc/filtron/rules.json +++ b/utils/templates/etc/filtron/rules.json @@ -1,107 +1,105 @@ -[{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":120, - "limit":19, - "actions":[ - { - "name":"block", - "params":{ - "message":"common rate limit exceeded" +[ + { "name": "suspiciously frequent IP", + "filters": [], + "interval": 600, + "limit": 30, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name":"log"} + ] + }, + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": 61, + "limit": 999, + "subrules": [ + { + "name": "roboagent limit", + "interval": 61, + "limit": 1, + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "IP limit", + "interval": 61, + "limit": 9, + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "interval": 121, + "limit": 2, + "stop": true, + "filters": [ + "Param:format=(csv|json|rss)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": 61, + "limit": 199, + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] } - } - ], - "subrules":[ - { - "name":"roboagent limit", - "interval":60, - "limit":3, - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby)" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" - } - } - ] - }, - { - "name":"botlimit", - "interval":60, - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" - } - } - ] - }, - { - "name":"IP limit", - "interval":60, - "limit":13, - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" - } - } - ] - }, - { - "name":"rss/json limit", - "interval":60, - "limit":13, - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" - } - } - ] - }, - { - "name":"useragent limit", - "interval":60, - "limit":13, - "aggregations":[ - "Header:User-Agent" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" - } - } - ] - } - ] -}] + ] + } +]