diff --git a/Makefile b/Makefile index fec004a5e..6577b374e 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,7 @@ test: test.pep8 test.unit test.sh test.robot test.sh: shellcheck -x utils/lib.sh shellcheck -x utils/filtron.sh + shellcheck -x utils/searx.sh test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/utils/filtron.sh b/utils/filtron.sh index 5c8a738b0..50cce6e65 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -192,7 +192,8 @@ assert_user() { rst_title "user $SERVICE_USER" section echo tee_stderr 1 < $max )); then @@ -333,7 +339,7 @@ install_template() { info_msg "install: ${template_file}" sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ "${template_file}" "${dst}" | prefix_stdout - break + break ;; "leave file unchanged") break @@ -343,12 +349,180 @@ install_template() { echo "// exit with CTRL-D" sudo -H -u "${owner}" -i $DIFF_CMD "${dst}" "${template_file}" - if ask_yn "did you edit ${template_file} to your needs?"; then - break - fi + if ask_yn "did you edit ${template_file} to your needs?"; then + break + fi ;; "diff files") $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout esac done } + + +# uWSGI +# ----- + +uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" + +uWSGI_restart() { + + # usage: uWSGI_restart() + + info_msg "restart uWSGI service" + sudo -H systemctl restart uwsgi +} + +uWSGI_install_app() { + + # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ... + + local do_eval="" + local CONF + + if [[ "$1" == "--no-eval" ]]; then + no_eval=$1; shift + fi + + for CONF in "$@"; do + install_template "$no_eval" "${CONF}" root root 644 + uWSGI_enable_app "$(basename "${CONF}")" + info_msg "enabled uWSGI app: $(basename "${CONF}")" + done + uWSGI_restart +} + +uWSGI_remove_app() { + + # usage: uWSGI_remove_app ... + + local CONF + for CONF in "$@"; do + uWSGI_disable_app "$(basename "${CONF}")" + rm -f "$CONF" + info_msg "removed uWSGI app: $(basename "${CONF}")" + done + uWSGI_restart +} + +# shellcheck disable=SC2164 +uWSGI_enable_app() { + + # usage: uWSGI_enable_app + + local CONF=$1 + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app missing arguments" + return 42 + fi + pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null + # shellcheck disable=SC2226 + ln -s "../apps-available/$(basename "${CONF}")" + info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" + popd >/dev/null +} + +uWSGI_disable_app() { + + # usage: uWSGI_disable_app + + local CONF=$1 + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app missing arguments" + return 42 + fi + + rm -f "${uWSGI_SETUP}/apps-enabled/$CONF" + info_msg "disabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" +} + +# distro's package manager +# ------------------------ +# +# FIXME: Arch Linux & RHEL should be added +# + +pkg_install() { + + # usage: TITEL='install foobar' pkg_install foopkg barpkg + + rst_title "${TITLE:-installation of packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be installed?" Yn 30; then + return 42 + fi + # shellcheck disable=SC2068 + apt-get install -y $@ + wait_key 30 +} + +pkg_remove() { + + # usage: TITEL='remove foobar' pkg_remove foopkg barpkg + + rst_title "${TITLE:-remove packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be removed (purge)?" Yn 30; then + return 42 + fi + apt-get purge --autoremove --ignore-missing -y "$@" + wait_key 30 +} + +pkg_is_installed() { + + # usage: pkg_is_install foopkg || pkg_install foopkg + + dpkg -l "$1" &> /dev/null + return $? +} + +# git tooling +# ----------- + +# shellcheck disable=SC2164 +git_clone() { + + # usage: + # + # git_clone [ []] + # git_clone [ []] + # + # First form uses $CACHE/ as destination folder, second form clones + # into . If repository is allready cloned, merge from origin and + # update working tree (if needed, the caller has to stash local changes). + # + # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin + # + + local url="$1" + local dest="$2" + local branch="$3" + local user="$4" + local prefix="" + + if [[ ! "${dest:0:1}" = "/" ]]; then + dest="$CACHE/$dest" + fi + + [[ -z $branch ]] && branch=master + [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}" + [[ -z $user ]] && prefix="sudo -H -u $user" + + if [[ -d "${dest}" ]] ; then + info_msg "already cloned: $dest" + pushd "${dest}" > /dev/null + $prefix git checkout -b "$(basename "$branch")" --track "$branch" + $prefix git pull --all + popd > /dev/null + + else + info_msg "clone into: $dest" + $prefix mkdir -p "$(dirname "$dest")" + pushd "${dest}" > /dev/null + git clone "$url" "$(basename "$dest")" + popd > /dev/null + fi +} diff --git a/utils/searx.sh b/utils/searx.sh new file mode 100755 index 000000000..0d47820b0 --- /dev/null +++ b/utils/searx.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh -*- +# shellcheck disable=SC2119 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_NAME}" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +SERVICE_HOME="/home/${SERVICE_USER}" + +SEARX_GIT_URL="https://github.com/asciimoo/searx.git" +SEARX_GIT_BRANCH="origin/master" + +# FIXME: Arch Linux & RHEL should be added + +SEARX_APT_PACKAGES="\ +libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \ + git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \ + libffi-dev libssl-dev" + +SEARX_VENV="${SEARX_HOME}/searx-venv" +SEARX_SRC="${SEARX_HOME}/searx-src" +SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" +SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${SEARX_UWSGI_APP}" +) + +# shellcheck disable=SC2034 +CONFIG_BACKUP_ENCRYPTED=( + "${SEARX_SETTINGS}" +) + +# ---------------------------------------------------------------------------- +usage(){ +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat < https://${SEARX_APACHE_DOMAIN}${SEARX_APACHE_URL}" + +} + +update_searx() { + rst_title "Update searx instance" + + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup +git stash push -m "BACKUP -- 'update server' at ($(date))" +git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH" +git pull "$SEARX_GIT_BRANCH" +${SEARX_SRC}/manage.sh update_packages +EOF + configure_searx + + rst_title "${SEARX_SETTINGS}" section + rstBlock 'Diff between new setting file (<) and backup (>):' + echo + diff "$SEARX_SETTINGS}" "${SEARX_SETTINGS}.backup" + + local action + choose_one action "What should happen to the settings file? " \ + "keep new configuration" \ + "revert to the old configuration (backup file)" \ + "start interactiv shell" + case $action in + "keep new configuration") + info_msg "continue using new settings file" + ;; + "revert to the old configuration (backup file)") + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS} +EOF + ;; + "start interactiv shell") + interactive_shell + ;; + esac + chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}" + + # shellcheck disable=SC2016 + rst_para 'Diff between local modified settings (<) and $SEARX_GIT_BRANCH branch (>):' + echo + git_diff + wait_key + uWSGI_restart +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + remove_service + wait_key + remove_user +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." + fi +} + +# shellcheck disable=SC2164 +clone_searx(){ + rst_title "Clone searx sources" section + echo + git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ + "$SEARX_GIT_BRANCH" "$SERVICE_USER" + + pushd "${SEARX_SRC}" > /dev/null + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd "${SEARX_SRC}" +git config user.email "$ADMIN_EMAIL" +git config user.name "$ADMIN_NAME" +git checkout "$SEARX_GIT_BRANCH" +EOF + popd > /dev/null +} + +create_venv(){ + rst_title "Create virtualenv (python)" section + + rst_para "Create venv in ${SEARX_VENV} and install needed python packages." + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_VENV}" +python3 -m venv "${SEARX_VENV}" +. ${SEARX_VENV}/bin/activate +${SEARX_SRC}/manage.sh update_packages +EOF + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile +EOF + +} + +configure_searx(){ + rst_title "Configure searx" section + rst_para "Setup searx config located at $SEARX_SETTINGS" + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS" +sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS" +EOF +} + +test_local_searx(){ + rstHeading "Testing searx instance localy" section + echo + tee_stderr 0.1 <&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +timeout 5 python3 searx/webapp.py & +sleep 1 +curl --location --verbose --head --insecure http://127.0.0.1:8888/ +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF + waitKEY +} + +install_searx_uwsgi() { + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app "$SEARX_UWSGI_APP" +} + +remove_searx_uwsgi() { + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +activate_service () { + rst_title "Activate $SERVICE_NAME (service)" section + uWSGI_enable_app "$SEARX_UWSGI_APP" +} + +deactivate_service () { + rst_title "De-Activate $SERVICE_NAME (service)" section + uWSGI_disable_app "$SEARX_UWSGI_APP" +} + +interactive_shell(){ + echo "// exit with CTRL-D" + sudo -H -u "${SERVICE_USER}" -i +} + +git_diff(){ + sudo -H -u "${SERVICE_USER}" -i <