utils/searx.sh: add script to install isolated searx service (WIP)

WIP: written from scratch / linted but untested

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

Makefile

@@ -79,6 +79,7 @@ test: test.pep8 test.unit test.sh test.robot
 test.sh:
 	shellcheck -x utils/lib.sh
 	shellcheck -x utils/filtron.sh
+	shellcheck -x utils/searx.sh
 
 test.pep8: pyenvinstall
 	$(PY_ENV_ACT); ./manage.sh pep8_check

utils/filtron.sh

@@ -192,7 +192,8 @@ assert_user() {
     rst_title "user $SERVICE_USER" section
     echo
     tee_stderr 1 <<EOF | bash | prefix_stdout
-sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME --group --gecos 'Filtron' $SERVICE_USER
+sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME \
+    --disabled-password --group --gecos 'Filtron' $SERVICE_USER
 sudo -H usermod -a -G shadow $SERVICE_USER
 groups $SERVICE_USER
 EOF

utils/lib.sh

@@ -2,6 +2,12 @@
 # -*- coding: utf-8; mode: sh -*-
 # shellcheck disable=SC2059,SC1117,SC2162,SC2004
 
+ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
+ADMIN_NAME="${ADMIN_NAME:-$USER}"
+
+ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}"
+ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}"
+
 if [[ -z "${REPO_ROOT}" ]]; then
     REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")
     while [ -h "${REPO_ROOT}" ] ; do
@@ -115,7 +121,7 @@ ask_yn() {
     esac
     echo
     while true; do
-	clean_stdin
+        clean_stdin
         printf "$1 ${choice} "
         # shellcheck disable=SC2086
         read -n1 $_t
@@ -240,7 +246,7 @@ choose_one() {
         fi
     done
     while true; do
-	clean_stdin
+        clean_stdin
         printf "$1 [$default] "
 
         if (( 10 > $max )); then
@@ -333,7 +339,7 @@ install_template() {
                 info_msg "install: ${template_file}"
                 sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
                      "${template_file}" "${dst}" | prefix_stdout
-		break
+                break
                 ;;
             "leave file unchanged")
                 break
@@ -343,12 +349,180 @@ install_template() {
                 echo "// exit with CTRL-D"
                 sudo -H -u "${owner}" -i
                 $DIFF_CMD "${dst}" "${template_file}"
-		if ask_yn "did you edit ${template_file} to your needs?"; then
+                if ask_yn "did you edit ${template_file} to your needs?"; then
-		    break
+                    break
-		fi
+                fi
                 ;;
             "diff files")
                 $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
         esac
     done
 }
+
+
+# uWSGI
+# -----
+
+uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}"
+
+uWSGI_restart() {
+
+    # usage:  uWSGI_restart()
+
+    info_msg "restart uWSGI service"
+    sudo -H systemctl restart uwsgi
+}
+
+uWSGI_install_app() {
+
+    # usage:  uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ...
+
+    local do_eval=""
+    local CONF
+
+    if [[ "$1" == "--no-eval" ]]; then
+        no_eval=$1; shift
+    fi
+
+    for CONF in "$@"; do
+        install_template "$no_eval" "${CONF}" root root 644
+        uWSGI_enable_app "$(basename "${CONF}")"
+        info_msg "enabled uWSGI app: $(basename "${CONF}")"
+    done
+    uWSGI_restart
+}
+
+uWSGI_remove_app() {
+
+    # usage:  uWSGI_remove_app <path.ini> ...
+
+    local CONF
+    for CONF in "$@"; do
+        uWSGI_disable_app "$(basename "${CONF}")"
+        rm -f "$CONF"
+        info_msg "removed uWSGI app: $(basename "${CONF}")"
+    done
+    uWSGI_restart
+}
+
+# shellcheck disable=SC2164
+uWSGI_enable_app() {
+
+    # usage:   uWSGI_enable_app <path.ini>
+
+    local CONF=$1
+    if [[ -z $CONF ]]; then
+        err_msg "uWSGI_enable_app missing arguments"
+        return 42
+    fi
+    pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null
+    # shellcheck disable=SC2226
+    ln -s "../apps-available/$(basename "${CONF}")"
+    info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"
+    popd >/dev/null
+}
+
+uWSGI_disable_app() {
+
+    # usage:   uWSGI_disable_app <path.ini>
+
+    local CONF=$1
+    if [[ -z $CONF ]]; then
+        err_msg "uWSGI_enable_app missing arguments"
+        return 42
+    fi
+
+    rm -f "${uWSGI_SETUP}/apps-enabled/$CONF"
+    info_msg "disabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"
+}
+
+# distro's package manager
+# ------------------------
+#
+# FIXME: Arch Linux & RHEL should be added
+#
+
+pkg_install() {
+
+    # usage: TITEL='install foobar' pkg_install foopkg barpkg
+
+    rst_title "${TITLE:-installation of packages}" section
+    echo -en "\npackage(s)::\n\n  $*\n" | $FMT
+
+    if ! ask_yn "Should packages be installed?" Yn 30; then
+        return 42
+    fi
+    # shellcheck disable=SC2068
+    apt-get install -y $@
+    wait_key 30
+}
+
+pkg_remove() {
+
+    # usage: TITEL='remove foobar' pkg_remove foopkg barpkg
+
+    rst_title "${TITLE:-remove packages}" section
+    echo -en "\npackage(s)::\n\n  $*\n" | $FMT
+
+    if ! ask_yn "Should packages be removed (purge)?" Yn 30; then
+        return 42
+    fi
+    apt-get purge --autoremove --ignore-missing -y "$@"
+    wait_key 30
+}
+
+pkg_is_installed() {
+
+    # usage: pkg_is_install foopkg || pkg_install foopkg
+
+    dpkg -l "$1" &> /dev/null
+    return $?
+}
+
+# git tooling
+# -----------
+
+# shellcheck disable=SC2164
+git_clone() {
+
+    # usage:
+    #
+    #    git_clone <url> <name> [<branch> [<user>]]
+    #    git_clone <url> <path> [<branch> [<user>]]
+    #
+    #  First form uses $CACHE/<name> as destination folder, second form clones
+    #  into <path>.  If repository is allready cloned, merge from origin and
+    #  update working tree (if needed, the caller has to stash local changes).
+    #
+    #    git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin
+    #
+
+    local url="$1"
+    local dest="$2"
+    local branch="$3"
+    local user="$4"
+    local prefix=""
+
+    if [[ ! "${dest:0:1}" = "/" ]]; then
+        dest="$CACHE/$dest"
+    fi
+
+    [[ -z $branch ]] && branch=master
+    [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}"
+    [[ -z $user ]] && prefix="sudo -H -u $user"
+
+    if [[ -d "${dest}" ]] ; then
+        info_msg "already cloned: $dest"
+        pushd "${dest}" > /dev/null
+        $prefix git checkout -b "$(basename "$branch")" --track "$branch"
+        $prefix git pull --all
+        popd > /dev/null
+
+    else
+        info_msg "clone into: $dest"
+        $prefix mkdir -p "$(dirname "$dest")"
+        pushd "${dest}" > /dev/null
+        git clone "$url" "$(basename "$dest")"
+        popd > /dev/null
+    fi
+}

utils/searx.sh

@@ -0,0 +1,349 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8; mode: sh -*-
+# shellcheck disable=SC2119
+
+# shellcheck source=utils/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+# ----------------------------------------------------------------------------
+# config
+# ----------------------------------------------------------------------------
+
+SERVICE_NAME="searx"
+SERVICE_USER="${SERVICE_NAME}"
+# shellcheck disable=SC2034
+SERVICE_GROUP="${SERVICE_USER}"
+SERVICE_HOME="/home/${SERVICE_USER}"
+
+SEARX_GIT_URL="https://github.com/asciimoo/searx.git"
+SEARX_GIT_BRANCH="origin/master"
+
+# FIXME: Arch Linux & RHEL should be added
+
+SEARX_APT_PACKAGES="\
+libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \
+  git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \
+  libffi-dev libssl-dev"
+
+SEARX_VENV="${SEARX_HOME}/searx-venv"
+SEARX_SRC="${SEARX_HOME}/searx-src"
+SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml"
+SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}"
+SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini"
+
+# shellcheck disable=SC2034
+CONFIG_FILES=(
+    "${SEARX_UWSGI_APP}"
+)
+
+# shellcheck disable=SC2034
+CONFIG_BACKUP_ENCRYPTED=(
+    "${SEARX_SETTINGS}"
+)
+
+# ----------------------------------------------------------------------------
+usage(){
+# ----------------------------------------------------------------------------
+
+    # shellcheck disable=SC1117
+    cat <<EOF
+
+usage:
+
+  $(basename "$0") shell
+  $(basename "$0") install    [all|user]
+  $(basename "$0") update     [searx]
+  $(basename "$0") remove     [all]
+  $(basename "$0") activate   [service]
+  $(basename "$0") deactivate [service]
+  $(basename "$0") show       [service]
+
+shell
+  start interactive shell from user ${SERVICE_USER}
+install / remove all
+  complete setup of searx service
+update searx
+  Update searx installation of user ${SERVICE_USER}
+activate
+  activate and start service daemon (systemd unit)
+deactivate service
+  stop and deactivate service daemon (systemd unit)
+install user
+  add service user '$SERVICE_USER' at $SERVICE_HOME
+show service
+  show service status and log
+EOF
+    [ ! -z ${1+x} ] &&  echo -e "$1"
+}
+
+main(){
+    rst_title "$SERVICE_NAME" part
+
+    local _usage="ERROR: unknown or missing $1 command $2"
+
+    case $1 in
+        --source-only)  ;;
+        -h|--help) usage; exit 0;;
+
+        shell)
+            sudo_or_exit
+            interactive_shell
+            ;;
+        show)
+            case $2 in
+                service)
+                    sudo_or_exit
+                    show_service
+                    ;;
+                *) usage "$_usage"; exit 42;;
+            esac ;;
+        install)
+            sudo_or_exit
+            case $2 in
+                all) install_all ;;
+                user) assert_user ;;
+                *) usage "$_usage"; exit 42;;
+            esac ;;
+        update)
+            sudo_or_exit
+            case $2 in
+                searx) update_searx;;
+                *) usage "$_usage"; exit 42;;
+            esac ;;
+        remove)
+            sudo_or_exit
+            case $2 in
+                all) remove_all;;
+                user) remove_user ;;
+                *) usage "$_usage"; exit 42;;
+            esac ;;
+        activate)
+            sudo_or_exit
+            case $2 in
+                service)  activate_service ;;
+                *) usage "$_usage"; exit 42;;
+            esac ;;
+        deactivate)
+            sudo_or_exit
+            case $2 in
+                service)  deactivate_service ;;
+                *) usage "$_usage"; exit 42;;
+            esac ;;
+        *) usage "ERROR: unknown or missing command $1"; exit 42;;
+    esac
+}
+
+_service_prefix="  |$SERVICE_USER| "
+
+install_all() {
+    rst_title "Install $SERVICE_NAME (service)"
+    pkg_install "$SEARX_APT_PACKAGES"
+    wait_key
+    assert_user
+    wait_key
+    clone_searx
+    wait_key
+    create_venv
+    wait_key
+    configure_searx
+    wait_key
+    test_local_searx
+    wait_key
+    install_searx_uwsgi
+    wait_key
+
+    # ToDo ...
+    # install_apache_site
+    # test_public_searx
+    # info_msg "searX --> https://${SEARX_APACHE_DOMAIN}${SEARX_APACHE_URL}"
+
+}
+
+update_searx() {
+    rst_title "Update searx instance"
+
+    echo
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+cd ${SEARX_SRC}
+cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup
+git stash push -m "BACKUP -- 'update server' at ($(date))"
+git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH"
+git pull "$SEARX_GIT_BRANCH"
+${SEARX_SRC}/manage.sh update_packages
+EOF
+    configure_searx
+
+    rst_title "${SEARX_SETTINGS}" section
+    rstBlock 'Diff between new setting file (<) and backup (>):'
+    echo
+    diff "$SEARX_SETTINGS}" "${SEARX_SETTINGS}.backup"
+
+    local action
+    choose_one action "What should happen to the settings file? " \
+           "keep new configuration" \
+           "revert to the old configuration (backup file)" \
+           "start interactiv shell"
+    case $action in
+        "keep new configuration")
+	    info_msg "continue using new settings file"
+	    ;;
+        "revert to the old configuration (backup file)")
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS}
+EOF
+	    ;;
+	"start interactiv shell")
+	    interactive_shell
+	    ;;
+    esac
+    chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}"
+
+    # shellcheck disable=SC2016
+    rst_para 'Diff between local modified settings (<) and $SEARX_GIT_BRANCH branch (>):'
+    echo
+    git_diff
+    wait_key
+    uWSGI_restart
+}
+
+remove_all() {
+    rst_title "De-Install $SERVICE_NAME (service)"
+    remove_service
+    wait_key
+    remove_user
+}
+
+assert_user() {
+    rst_title "user $SERVICE_USER" section
+    echo
+    tee_stderr 1 <<EOF | bash | prefix_stdout
+sudo -H adduser --shell /bin/bash --system --home "$SERVICE_HOME" \
+  --disabled-password --group --gecos 'searx' $SERVICE_USER
+sudo -H usermod -a -G shadow $SERVICE_USER
+groups $SERVICE_USER
+EOF
+    #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
+    #export SERVICE_HOME
+    #echo "export SERVICE_HOME=$SERVICE_HOME"
+}
+
+remove_user() {
+    rst_title "Drop $SERVICE_USER HOME" section
+    if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then
+        userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout
+    else
+        rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged."
+    fi
+}
+
+# shellcheck disable=SC2164
+clone_searx(){
+    rst_title "Clone searx sources" section
+    echo
+    git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \
+	      "$SEARX_GIT_BRANCH" "$SERVICE_USER"
+
+    pushd "${SEARX_SRC}" > /dev/null
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
+cd "${SEARX_SRC}"
+git config user.email "$ADMIN_EMAIL"
+git config user.name "$ADMIN_NAME"
+git checkout "$SEARX_GIT_BRANCH"
+EOF
+    popd > /dev/null
+}
+
+create_venv(){
+    rst_title "Create virtualenv (python)" section
+
+    rst_para "Create venv in ${SEARX_VENV} and install needed python packages."
+    echo
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+rm -rf "${SEARX_VENV}"
+python3 -m venv "${SEARX_VENV}"
+. ${SEARX_VENV}/bin/activate
+${SEARX_SRC}/manage.sh update_packages
+EOF
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \
+  || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile
+EOF
+
+}
+
+configure_searx(){
+    rst_title "Configure searx" section
+    rst_para "Setup searx config located at $SEARX_SETTINGS"
+    echo
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+cd ${SEARX_SRC}
+sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS"
+sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS"
+EOF
+}
+
+test_local_searx(){
+    rstHeading "Testing searx instance localy" section
+    echo
+    tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 |  prefix_stdout "$_service_prefix"
+cd ${SEARX_SRC}
+sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS"
+timeout 5 python3 searx/webapp.py &
+sleep 1
+curl --location --verbose --head --insecure http://127.0.0.1:8888/
+sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS"
+EOF
+    waitKEY
+}
+
+install_searx_uwsgi() {
+    rst_title "Install searx's uWSGI app (searx.ini)" section
+    echo
+    uWSGI_install_app "$SEARX_UWSGI_APP"
+}
+
+remove_searx_uwsgi() {
+    rst_title "Remove searx's uWSGI app (searx.ini)" section
+    echo
+    uWSGI_remove_app "$SEARX_UWSGI_APP"
+}
+
+activate_service () {
+    rst_title "Activate $SERVICE_NAME (service)" section
+    uWSGI_enable_app "$SEARX_UWSGI_APP"
+}
+
+deactivate_service () {
+    rst_title "De-Activate $SERVICE_NAME (service)" section
+    uWSGI_disable_app "$SEARX_UWSGI_APP"
+}
+
+interactive_shell(){
+    echo "// exit with CTRL-D"
+    sudo -H -u "${SERVICE_USER}" -i
+}
+
+git_diff(){
+    sudo -H -u "${SERVICE_USER}" -i <<EOF
+cd ${SEARX_REPO_FOLDER}
+git --no-pager diff
+EOF
+}
+
+show_service () {
+    rst_title "service status & log"
+    echo
+    systemctl status uwsgi.service
+    echo
+    read -r -s -n1 -t 5  -p "// use CTRL-C to stop monitoring the log"
+    echo
+    while true;  do
+        trap break 2
+        journalctl -f -u uwsgi.service
+    done
+    return 0
+}
+
+# ----------------------------------------------------------------------------
+main "$@"
+# ----------------------------------------------------------------------------