Changes from the installation tests on (all) LXC containers.

Tested and fixed HTTP & uWSGI installation on:

  ubu1604 ubu1804 ubu1910 ubu2004 fedora31 archlinux

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

docs/utils/index.rst

@@ -19,20 +19,28 @@ developers.
 
 .. _toolboxing common:
 
-Common commands
+Common commands & environment
-===============
+=============================
 
 Scripts to maintain services often dispose of common commands and environments.
 
-``shell``:
+``shell`` : command
   Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
   troubleshooting.
 
-``inspect service``:
+``inspect service`` : command
   Shows status and log of the service, most often you have a option to enable
   more verbose debug logs.  Very helpful for debugging, but be careful not to
   enable debugging in a production environment!
 
+``FORCE_TIMEOUT`` : environment
+  Sets timeout for interactive prompts. If you want to run a script in batch
+  job, with defaults choices, set ``FORCE_TIMEOUT=0``.  By example; to install a
+  reverse proxy for filtron on all containers of the :ref:`searx suite
+  <lxc-searx.env>` use ::
+
+    sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
+ 
 .. _toolboxing setup:
 
 Tooling box setup

docs/utils/lxc.sh.rst

@@ -88,6 +88,10 @@ WEB-Browser::
   [searx-fedora31]  INFO:  (eth0) filtron:    http://n.n.n.18:4004/
   [searx-archlinux]  INFO:  (eth0) filtron:    http://n.n.n.12:4004/
 
+To install a reverse proxy for filtron and morty use::
+
+    sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
+    sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh apache install
 
 Running commands
 ================

utils/filtron.sh

@@ -23,7 +23,6 @@ FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
 [[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
 
 FILTRON_ETC="/etc/filtron"
-
 FILTRON_RULES="$FILTRON_ETC/rules.json"
 
 FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
@@ -447,7 +446,8 @@ This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SIT
         install_apache
     fi
 
-    echo
+    "${REPO_ROOT}/utils/searx.sh" install uwsgi
+
     apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
 
     info_msg "testing public url .."
@@ -465,11 +465,12 @@ This removes apache site ${APACHE_FILTRON_SITE}."
 
     ! apache_is_installed && err_msg "Apache is not installed."
 
-    if ! ask_yn "Do you really want to continue?"; then
+    if ! ask_yn "Do you really want to continue?" Yn; then
         return
     fi
 
     apache_remove_site "$APACHE_FILTRON_SITE"
+
 }
 
 rst-doc() {

utils/lib.sh

@@ -389,7 +389,7 @@ install_template() {
     local chmod="${pos_args[4]-644}"
 
     info_msg "install (eval=$do_eval): ${dst}"
-    [[ -n $variant ]] && info_msg "variant: ${variant}"
+    [[ -n $variant ]] && info_msg "variant --> ${variant}"
 
     if [[ ! -f "${template_origin}" ]] ; then
         err_msg "${template_origin} does not exists"
@@ -777,6 +777,7 @@ apache_dissable_site() {
             ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
             ;;
     esac
+    apache_reload
 }
 
 # uWSGI
@@ -846,7 +847,7 @@ uWSGI_restart() {
             if uWSGI_app_available "${CONF}"; then
                 systemctl restart "uwsgi@${CONF%.*}"
             else
-                info_msg "in systemd template mode: ${CONF} not installed (nothing to restart)"
+                info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"
             fi
             ;;
         fedora-*)
@@ -854,7 +855,7 @@ uWSGI_restart() {
             if uWSGI_app_enabled "${CONF}"; then
                 touch "${uWSGI_APPS_ENABLED}/${CONF}"
             else
-                info_msg "in uWSGI emperor mode: ${CONF} not installed (nothing to restart)"
+                info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
             fi
             ;;
         *)
@@ -864,6 +865,32 @@ uWSGI_restart() {
     esac
 }
 
+uWSGI_prepare_app() {
+
+    # usage:  uWSGI_prepare_app <myapp.ini>
+
+    local APP="${1%.*}"
+    if [[ -z $APP ]]; then
+        err_msg "uWSGI_prepare_app: missing arguments"
+        return 42
+    fi
+
+    case $DIST_ID-$DIST_VERS in
+        fedora-*)
+            # in emperor mode, the uwsgi user is the owner of the sockets
+            info_msg "prepare (uwsgi:uwsgi)  /run/uwsgi/app/${APP}"
+            mkdir -p "/run/uwsgi/app/${APP}"
+            chown -R "uwsgi:uwsgi"  "/run/uwsgi/app/${APP}"
+            ;;
+        *)
+            info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP})  /run/uwsgi/app/${APP}"
+            mkdir -p "/run/uwsgi/app/${APP}"
+            chown -R "${SERVICE_USER}:${SERVICE_GROUP}"  "/run/uwsgi/app/${APP}"
+            ;;
+    esac
+}
+
+
 uWSGI_app_available() {
     # usage:  uWSGI_app_available <myapp.ini>
     local CONF="$1"
@@ -888,6 +915,7 @@ uWSGI_install_app() {
             *)  pos_args+=("$i");;
         esac
     done
+    uWSGI_prepare_app "${pos_args[1]}"
     mkdir -p "${uWSGI_APPS_AVAILABLE}"
     install_template "${template_opts[@]}" \
                      "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
@@ -1281,3 +1309,30 @@ global_IPs(){
 
     ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'
 }
+
+primary_ip() {
+
+    case $DIST_ID in
+        arch)
+            echo "$(ip -o addr show \
+            | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
+            | head -n 1)"
+            ;;
+        *)  echo "$(hostname -I | cut -d' ' -f1)" ;;
+    esac
+}
+
+# URL
+# ---
+
+url_replace_hostname(){
+
+    # usage:  url_replace_hostname <url> <new hostname>
+
+    # to replace hostname by primary IP::
+    #
+    #   url_replace_hostname http://searx-ubu1604/morty $(primary_ip)
+    #   http://10.246.86.250/morty
+
+    echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|"
+}

utils/lxc.sh

@@ -438,7 +438,6 @@ lxc_cmd() {
         else
             info_msg "lxc $* $i"
             lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
-            echo
         fi
     done
 }

utils/morty.sh

@@ -14,12 +14,16 @@ in_container && lxc_set_suite_env
 # config
 # ----------------------------------------------------------------------------
 
-PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
-PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
-PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"
-PUBLIC_URL_MORTY="$(echo "$PUBLIC_URL" |  sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
-
 MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
+PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty}"
+
+SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
+PUBLIC_URL_MORTY="$(echo "$SEARX_URL" |  sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}"
+if in_container; then
+    # container hostnames do not have a DNS entry, use primary IP
+    PUBLIC_URL_MORTY="$(url_replace_hostname "$PUBLIC_URL_MORTY" "$(primary_ip)")"
+fi
+
 # shellcheck disable=SC2034
 MORTY_TIMEOUT=5
 
@@ -425,7 +429,7 @@ This removes apache site ${APACHE_MORTY_SITE}."
 
     ! apache_is_installed && err_msg "Apache is not installed."
 
-    if ! ask_yn "Do you really want to continue?"; then
+    if ! ask_yn "Do you really want to continue?" Yn; then
         return
     fi
 

utils/searx.sh

@@ -748,6 +748,10 @@ excessively bot queries."
 
     apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}"
 
+    rst_title "Install searx's uWSGI app (searx.ini)" section
+    echo
+    uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP"
+
     if ! service_is_available "${PUBLIC_URL}"; then
         err_msg "Public service at ${PUBLIC_URL} is not available!"
     fi
@@ -762,11 +766,15 @@ This removes apache site ${APACHE_SEARX_SITE}."
 
     ! apache_is_installed && err_msg "Apache is not installed."
 
-    if ! ask_yn "Do you really want to continue?"; then
+    if ! ask_yn "Do you really want to continue?" Yn; then
         return
     fi
 
     apache_remove_site "${APACHE_SEARX_SITE}"
+
+    rst_title "Remove searx's uWSGI app (searx.ini)" section
+    echo
+    uWSGI_remove_app "$SEARX_UWSGI_APP"
 }
 
 rst-doc() {

utils/templates/etc/httpd/sites-available/morty.conf

@@ -2,7 +2,7 @@
 
 LoadModule headers_module       ${APACHE_MODULES}/mod_headers.so
 LoadModule proxy_module         ${APACHE_MODULES}/mod_proxy.so
-LoadModule proxy_module         ${APACHE_MODULES}/mod_proxy_http.so
+LoadModule proxy_http_module    ${APACHE_MODULES}/mod_proxy_http.so
 #LoadModule setenvif_module      ${APACHE_MODULES}/mod_setenvif.so
 
 # SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog

utils/templates/etc/httpd/sites-available/searx.conf:filtron

@@ -2,7 +2,7 @@
 
 LoadModule headers_module       ${APACHE_MODULES}/mod_headers.so
 LoadModule proxy_module         ${APACHE_MODULES}/mod_proxy.so
-LoadModule proxy_module         ${APACHE_MODULES}/mod_proxy_http.so
+LoadModule proxy_http_module    ${APACHE_MODULES}/mod_proxy_http.so
 #LoadModule setenvif_module      ${APACHE_MODULES}/mod_setenvif.so
 
 # SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog

utils/templates/etc/uwsgi/apps-archlinux/searx.ini

@@ -74,7 +74,7 @@ http = ${SEARX_INTERNAL_HTTP}
 #
 # On some distributions you need to create the app folder for the sockets::
 #
-#   mkdir -p /run/uwsgi/app/searx/socket
+#   mkdir -p /run/uwsgi/app/searx
-#   chmod -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx/socket
+#   chown -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx
 #
 # socket = /run/uwsgi/app/searx/socket

utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket

@@ -0,0 +1,80 @@
+[uwsgi]
+
+# uWSGI core
+# ----------
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
+
+# Who will run the code
+uid = ${SERVICE_USER}
+gid = ${SERVICE_GROUP}
+
+# chdir to specified directory before apps loading
+chdir = ${SEARX_SRC}/searx
+
+# searx configuration (settings.yml)
+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
+
+# disable logging for privacy
+logger = systemd
+disable-logging = true
+
+# The right granted on the created socket
+chmod-socket = 666
+
+# Plugin to use and interpretor config
+single-interpreter = true
+
+# enable master process
+master = true
+
+# load apps in each worker instead of the master
+lazy-apps = true
+
+# load uWSGI plugins
+plugin = python
+
+# By default the Python plugin does not initialize the GIL.  This means your
+# app-generated threads will not run.  If you need threads, remember to enable
+# them with enable-threads.  Running uWSGI in multithreading mode (with the
+# threads options) will automatically enable threading support. This *strange*
+# default behaviour is for performance reasons.
+enable-threads = true
+
+
+# plugin: python
+# --------------
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
+
+# load a WSGI module
+module = searx.webapp
+
+# set PYTHONHOME/virtualenv
+virtualenv = ${SEARX_PYENV}
+
+# add directory (or glob) to pythonpath
+pythonpath = ${SEARX_SRC}
+
+
+# speak to upstream
+# -----------------
+#
+# Activate the 'http' configuration for filtron or activate the 'socket'
+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
+
+# using IP:
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
+
+# http = ${SEARX_INTERNAL_HTTP}
+
+# using unix-sockets:
+#
+# On some distributions you need to create the app folder for the sockets::
+#
+#   mkdir -p /run/uwsgi/app/searx
+#   chown -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx
+#
+socket = /run/uwsgi/app/searx/socket

utils/templates/etc/uwsgi/apps-available/searx.ini

@@ -73,7 +73,7 @@ http = ${SEARX_INTERNAL_HTTP}
 #
 # On some distributions you need to create the app folder for the sockets::
 #
-#   mkdir -p /run/uwsgi/app/searx/socket
+#   mkdir -p /run/uwsgi/app/searx
-#   chmod -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx/socket
+#   chmod -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx
 #
 # socket = /run/uwsgi/app/searx/socket

utils/templates/etc/uwsgi/apps-available/searx.ini:socket

@@ -0,0 +1,79 @@
+[uwsgi]
+
+# uWSGI core
+# ----------
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
+
+# Who will run the code
+uid = ${SERVICE_USER}
+gid = ${SERVICE_GROUP}
+
+# chdir to specified directory before apps loading
+chdir = ${SEARX_SRC}/searx
+
+# searx configuration (settings.yml)
+env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH}
+
+# disable logging for privacy
+disable-logging = true
+
+# The right granted on the created socket
+chmod-socket = 666
+
+# Plugin to use and interpretor config
+single-interpreter = true
+
+# enable master process
+master = true
+
+# load apps in each worker instead of the master
+lazy-apps = true
+
+# load uWSGI plugins
+plugin = python3,http
+
+# By default the Python plugin does not initialize the GIL.  This means your
+# app-generated threads will not run.  If you need threads, remember to enable
+# them with enable-threads.  Running uWSGI in multithreading mode (with the
+# threads options) will automatically enable threading support. This *strange*
+# default behaviour is for performance reasons.
+enable-threads = true
+
+
+# plugin: python
+# --------------
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python
+
+# load a WSGI module
+module = searx.webapp
+
+# set PYTHONHOME/virtualenv
+virtualenv = ${SEARX_PYENV}
+
+# add directory (or glob) to pythonpath
+pythonpath = ${SEARX_SRC}
+
+
+# speak to upstream
+# -----------------
+#
+# Activate the 'http' configuration for filtron or activate the 'socket'
+# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
+
+# using IP:
+#
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
+
+# http = ${SEARX_INTERNAL_HTTP}
+
+# using unix-sockets:
+#
+# On some distributions you need to create the app folder for the sockets::
+#
+#   mkdir -p /run/uwsgi/app/searx
+#   chown -R ${SERVICE_USER}:${SERVICE_GROUP}  /run/uwsgi/app/searx
+#
+socket = /run/uwsgi/app/searx/socket