From a4437c47ac0bd22cd7f5aaa8e7895cdd8e93a317 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sun, 2 Feb 2020 18:14:10 +0100 Subject: [PATCH] utils/morty.sh: add script to install morty result proxy Signed-off-by: Markus Heiser --- utils/dot_config | 14 +- utils/filtron.sh | 43 +- utils/lib.sh | 28 +- utils/morty.sh | 382 ++++++++++++++++++ utils/searx.sh | 36 +- .../etc/apache2/sites-available/morty.conf | 23 ++ .../lib/systemd/system/morty.service | 29 ++ 7 files changed, 506 insertions(+), 49 deletions(-) create mode 100755 utils/morty.sh create mode 100644 utils/templates/etc/apache2/sites-available/morty.conf create mode 100644 utils/templates/lib/systemd/system/morty.service diff --git a/utils/dot_config b/utils/dot_config index 0d6758a18..1ddfa868c 100644 --- a/utils/dot_config +++ b/utils/dot_config @@ -1,9 +1,17 @@ # -*- coding: utf-8; mode: sh -*- # SPDX-License-Identifier: AGPL-3.0-or-later # -# environment used by utils scripts like filtron.sh or searx.sh -# +# This environment is used by ./utils scripts like filtron.sh or searx.sh. The +# default values are *most flexible* and *best maintained*, you normally not +# need to change them. Before you change any value here you have to uninstall +# any previous installation. It is recommended to backup your changes simply by +# adding them to you local brand (git branch). -# the public URL of the searx instance +# The public URL of the searx instance PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" + +# Run all services by one account, but be aware that removing discrete +# components might conflict! **experimental** +# +# SERVICE_USER=service_account42 diff --git a/utils/filtron.sh b/utils/filtron.sh index 2aa8cf4f5..dd49a3d1e 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -24,9 +24,11 @@ FILTRON_LISTEN="127.0.0.1:4004" FILTRON_TARGET="127.0.0.1:8888" SERVICE_NAME="filtron" -SERVICE_USER="${SERVICE_NAME}" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" SERVICE_HOME="/home/${SERVICE_USER}" SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" # shellcheck disable=SC2034 SERVICE_GROUP="${SERVICE_USER}" @@ -52,7 +54,7 @@ usage() { # shellcheck disable=SC1117 cat <&1 | prefix_stdout if service_is_available "${PUBLIC_URL}"; then @@ -231,7 +232,7 @@ export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin EOF echo "Environment $GO_ENV has been setup." - tee_stderr <> ~/.profile EOF } @@ -241,10 +242,12 @@ filtron_is_installed() { [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] } +_svcpr=" |${SERVICE_USER}| " + install_filtron() { rst_title "Install filtron in user's ~/go-apps" section echo - tee_stderr <&1 | prefix_stdout "$_service_prefix" + tee_stderr <&1 | prefix_stdout "$_svcpr" go get -v -u github.com/asciimoo/filtron EOF install_template --no-eval "$FILTRON_RULES" root root 644 @@ -253,7 +256,7 @@ EOF update_filtron() { rst_title "Update filtron" section echo - tee_stderr <&1 | prefix_stdout "$_service_prefix" + tee_stderr <&1 | prefix_stdout "$_svcpr" go get -v -u github.com/asciimoo/filtron EOF } @@ -301,12 +304,14 @@ EOF err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" fi - if ! service_is_available ""http://${FILTRON_TARGET}"" ; then + if service_is_available ""http://${FILTRON_TARGET}"" ; then info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" fi if ! service_is_available "${PUBLIC_URL}"; then err_msg "Public service at ${PUBLIC_URL} is not available!" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key fi local _debug_on @@ -316,15 +321,17 @@ EOF fi echo - systemctl --no-pager -l status filtron.service + systemctl --no-pager -l status "${SERVICE_NAME}" echo + + info_msg "public URL --> ${PUBLIC_URL}" # shellcheck disable=SC2059 printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" read -r -s -n1 -t 2 echo while true; do trap break 2 - journalctl -f -u filtron + journalctl -f -u "${SERVICE_NAME}" done if [[ $_debug_on == 1 ]]; then diff --git a/utils/lib.sh b/utils/lib.sh index 4a77671a1..818477fc3 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -112,9 +112,9 @@ rst_title() { # usage: rst_title [part|chapter|section] case ${2-chapter} in - part) printf "\n${_BGreen}${1//?/=}\n$1\n${1//?/=}${_creset}\n";; - chapter) printf "\n${_BGreen}${1}\n${1//?/=}${_creset}\n";; - section) printf "\n${_BGreen}${1}\n${1//?/-}${_creset}\n";; + part) printf "\n${_BGreen}${1//?/=}\n${_BCyan}${1}${_BGreen}\n${1//?/=}${_creset}\n";; + chapter) printf "\n${_BCyan}${1}\n${_BGreen}${1//?/=}${_creset}\n";; + section) printf "\n${_BCyan}${1}\n${_BGreen}${1//?/-}${_creset}\n";; *) err_msg "invalid argument '${2}' in line $(caller)" return 42 @@ -169,7 +169,9 @@ ask_yn() { local _t=$3 [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT [[ ! -z $_t ]] && _t="-t $_t" - case "${2}" in + case "${FORCE_SELECTION:-${2}}" in + Y) return ${EXIT_YES} ;; + N) return ${EXIT_NO} ;; Yn) local exit_val=${EXIT_YES} local choice="[${_BGreen}YES${_creset}/no]" @@ -229,7 +231,7 @@ tee_stderr () { prefix_stdout () { # usage: | prefix_stdout [prefix] - local prefix=" | " + local prefix="${_BYellow}-->|${_creset}" if [[ ! -z $1 ]] ; then prefix="${_BYellow}$1${_creset}"; fi @@ -433,7 +435,7 @@ install_template() { ;; "interactiv shell") echo "// edit ${dst} to your needs" - echo "// exit with ${_BCyan}CTRL-D${_creset}" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" sudo -H -u "${owner}" -i $DIFF_CMD "${dst}" "${template_file}" echo @@ -487,14 +489,14 @@ install_go() { # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" - local _service_prefix=" |${3}| " + local _svcpr=" |${3}| " rst_title "Install Go in user's HOME" section rst_para "download and install go binary .." cache_download "${1}" "${2}" - tee_stderr 0.1 <&1 | prefix_stdout @@ -845,14 +847,14 @@ git_clone() { if [[ -d "${dest}" ]] ; then info_msg "already cloned: $dest" - tee_stderr 0.1 <&1 | prefix_stdout " |$user| " + tee_stderr 0.1 <&1 | prefix_stdout " |$user| " cd "${dest}" git checkout -m -B "$branch" --track "$remote/$branch" git pull --all EOF else info_msg "clone into: $dest" - tee_stderr 0.1 <&1 | prefix_stdout " |$user| " + tee_stderr 0.1 <&1 | prefix_stdout " |$user| " mkdir -p "$(dirname "$dest")" cd "$(dirname "$dest")" git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")" diff --git a/utils/morty.sh b/utils/morty.sh new file mode 100755 index 000000000..49f7e2724 --- /dev/null +++ b/utils/morty.sh @@ -0,0 +1,382 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL_PATH_MORTY="/morty" +PUBLIC_URL_MORTY="$(dirname ${PUBLIC_URL})${PUBLIC_URL_PATH_MORTY}" + +MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" +MORTY_TIMEOUT=5 + +SERVICE_NAME="morty" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME="/home/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +SERVICE_ENV_DEBUG=false + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# shellcheck disable=SC2034 +CONFIG_FILES=() + +# Apache Settings + +APACHE_MORTY_SITE="morty.conf" + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat < "$GO_ENV" <> ~/.profile +EOF +} + +morty_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/morty ]] +} + +_svcpr=" |${SERVICE_USER}| " + +install_morty() { + rst_title "Install morty in user's ~/go-apps" section + echo + tee_stderr <&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +update_morty() { + rst_title "Update morty" section + echo + tee_stderr <&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +set_service_env_debug() { + + # usage: set_service_env_debug [false|true] + + local SERVICE_ENV_DEBUG="${1:-false}" + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + fi +} + +inspect_service() { + + rst_title "service status & log" + + cat < http://${MORTY_LISTEN}" + info_msg "public URL --> ${PUBLIC_URL_MORTY}" + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 2 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + FORCE_SELECTION=Y disable_debug + fi + return 0 +} + + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "Enabling debug option needs to reinstall systemd service!" + set_service_env_debug true +} + +disable_debug() { + info_msg "Disabling debug option needs to reinstall systemd service!" + set_service_env_debug false +} + +install_apache_site() { + + rst_title "Install Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + a2enmod headers + a2enmod proxy + a2enmod proxy_http + + echo + apache_install_site "${APACHE_MORTY_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This removes apache site ${APACHE_MORTY_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "$APACHE_MORTY_SITE" +} +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searx.sh b/utils/searx.sh index 47aad286b..204386fa8 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -17,10 +17,11 @@ SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \ SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \ | sed -e 's,^.*://\([^\:/]*\).*,\1,g') }" -SERVICE_USER="searx" +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME="/home/${SERVICE_USER}" # shellcheck disable=SC2034 SERVICE_GROUP="${SERVICE_USER}" -SERVICE_HOME="/home/${SERVICE_USER}" SEARX_INTERNAL_URL="127.0.0.1:8888" SEARX_GIT_URL="https://github.com/asciimoo/searx.git" @@ -65,7 +66,7 @@ usage() { # shellcheck disable=SC1117 cat <&1 | prefix_stdout "$_service_prefix" cd ${SEARX_SRC} @@ -500,8 +501,10 @@ EOF uWSGI_app_available "$SEARX_UWSGI_APP" \ || err_msg "uWSGI app $SEARX_UWSGI_APP not available!" - if ! service_is_available "http://$SEARX_INTERNAL_URL"; then - err_msg "uWSGI app (service) at http://$SEARX_INTERNAL_URL is not available!" + if ! service_is_available "http://${SEARX_INTERNAL_URL}"; then + err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_URL} is not available!" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key fi if ! service_is_available "${PUBLIC_URL}"; then @@ -514,15 +517,18 @@ EOF _debug_on=1 fi echo - systemctl --no-pager -l status uwsgi.service + systemctl --no-pager -l status "${SERVICE_NAME}" echo + # shellcheck disable=SC2059 + info_msg "public URL --> ${PUBLIC_URL}" + info_msg "internal URL --> http://${SEARX_INTERNAL_URL}" printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" read -r -s -n1 -t 2 echo while true; do trap break 2 - #journalctl -f -u uwsgi.service + #journalctl -f -u "${SERVICE_NAME}" tail -f /var/log/uwsgi/app/searx.log done diff --git a/utils/templates/etc/apache2/sites-available/morty.conf b/utils/templates/etc/apache2/sites-available/morty.conf new file mode 100644 index 000000000..231b3fb79 --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/morty.conf @@ -0,0 +1,23 @@ +# -*- coding: utf-8; mode: apache -*- + +ProxyPreserveHost On + + + + Require all granted + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPass http://${MORTY_LISTEN} + RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} + + # In Apache it seems, that setting HTTP_HOST header direct here does have no + # effect. I needed to set 'ProxyPreserveHost On' (see above). HTTP_HOST is + # needed by searx to render correct *Search URL* in the *Link* box and + # *saved preference*. + + # RequestHeader set Host ${PUBLIC_URL_PATH_MORTY} + + diff --git a/utils/templates/lib/systemd/system/morty.service b/utils/templates/lib/systemd/system/morty.service new file mode 100644 index 000000000..d463c5097 --- /dev/null +++ b/utils/templates/lib/systemd/system/morty.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT} + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target