searxng/manage

#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck disable=SC2031

# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"

# shellcheck source=utils/lib_static.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"

# config

PYOBJECTS="searx"
PY_SETUP_EXTRAS='[test]'
GECKODRIVER_VERSION="v0.28.0"
# SPHINXOPTS=

pylint.FILES() {

    # List files tagged by comment:
    #
    #   # lint: pylint
    #
    # These py files are linted by test.pylint(), all other files are linted by
    # test.pep8()

    grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searx_extra tests
}

YAMLLINT_FILES=()
while IFS= read -r line; do
   YAMLLINT_FILES+=("$line")
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searx/*.yml')"

PYLINT_SEARX_DISABLE_OPTION="\
I,C,R,\
W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
E1136"
PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases,logger,categories"
PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"

help() {
    cat <<EOF
buildenv:
  rebuild ./utils/brand.env
weblate.:
  push.translations: push translation changes from SearXNG to Weblate's counterpart
  to.translations: Update 'translations' branch with last additions from Weblate.
data.:
  all       : update searx/languages.py and ./data/*
  languages : update searx/data/engines_languages.json & searx/languages.py
  useragents: update searx/data/useragents.json with the most recent versions of Firefox.
docs.:
  html      : build HTML documentation
  live      : autobuild HTML documentation while editing
  gh-pages  : deploy on gh-pages branch
  prebuild  : build reST include files (./${DOCS_BUILD}/includes)
  clean     : clean documentation build
docker.:
  build     : build docker image
  push      : build and push docker image
gecko.driver:
  download & install geckodriver if not already installed (required for
  robot_tests)
node.:
  env       : download & install npm dependencies locally
  clean     : drop npm installations
py.:
  build     : Build python packages at ./${PYDIST}
  clean     : delete virtualenv and intermediate py files
pyenv.:
  install   : developer install of searx into virtualenv
  uninstall : uninstall developer installation
  cmd ...   : run command ... in virtualenv
  OK        : test if virtualenv is OK
pypi.upload:
  Upload python packages to PyPi (to test use pypi.upload.test)
test.:
  pylint    : lint PYLINT_FILES, searx/engines, searx & tests
  pep8      : pycodestyle (pep8) for all files except PYLINT_FILES
  unit      : run unit tests
  coverage  : run unit tests with coverage
  robot     : run robot test
  clean     : clean intermediate test stuff
themes.:
  all       : build all themes
  oscar     : build oscar theme
  simple    : build simple theme
pygments.:
  less      : build LESS files for pygments
EOF
    static_help
}


if [ "$VERBOSE" = "1" ]; then
    SPHINX_VERBOSE="-v"
    PYLINT_VERBOSE="-v"
fi

# needed by sphinx-docs
export DOCS_BUILD

buildenv() {

    # settings file from repository's working tree are used by default
    SEARX_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml"

    if [ -r '/etc/searx/settings.yml' ]; then
        if ask_yn "should settings read from: /etc/searx/settings.yml"; then
            SEARX_SETTINGS_PATH='/etc/searx/settings.yml'
        fi
    fi
    export SEARX_SETTINGS_PATH
    (
        set -e
        SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
            | prefix_stdout "${_Blue}BUILDENV${_creset}  "
    )
    return "${PIPESTATUS[0]}"
}

TRANSLATIONS_WORKTREE="$CACHE/translations"

weblate.translations.worktree() {

    # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
    # 'translations' from Weblate's counterpart (weblate) of the SearXNG
    # (origin).
    #
    #     remote weblate https://weblate.bubu1.eu/git/searxng/searxng/

    (   set -e
        if ! git remote get-url weblate 2> /dev/null; then
            git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
        fi
        if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
            pushd .
            cd "${TRANSLATIONS_WORKTREE}"
            git reset --hard HEAD
            git pull origin translations
            popd
        else
            mkdir -p "${TRANSLATIONS_WORKTREE}"
            git worktree add "${TRANSLATIONS_WORKTREE}" translations
        fi
    )
}

weblate.to.translations() {

    # Update 'translations' branch of SearXNG (origin) with last additions from
    # Weblate.

    # 1. Check if Weblate is locked, if not die with error message
    # 2. On Weblate's counterpart (weblate), pull master and translations branch
    #    from SearXNG (origin).
    # 3. Commit changes made in a Weblate object on Weblate's counterpart
    #    (weblate).
    # 4. In translations worktree, merge changes of branch 'translations' from
    #    remote 'weblate' and push it on branch 'translations' of 'origin'

    (   set -e
        pyenv.activate
        if [ "$(wlc lock-status)" != "locked: True" ]; then
            die 1 "weblate must be locked, currently: $(wlc lock-status)"
        fi
        # weblate: commit pending changes
        wlc pull
        wlc commit

        # get the translations in a worktree
        weblate.translations.worktree

        pushd "${TRANSLATIONS_WORKTREE}"
        git remote update weblate
        git merge weblate/translations
        git push
        popd
    )
    dump_return $?
}

weblate.translations.commit() {

    # Update 'translations' branch of SearXNG (origin) with last additions from
    # Weblate.  Copy the changes to the master branch, compile translations and
    # create a commit in the local branch (master)

    local existing_commit_hash commit_body commit_message exitcode
    (   set -e
        pyenv.activate
        # lock change on weblate
        wlc lock

        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
        weblate.translations.worktree
        existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1  --pretty=format:'%h')

        # pull weblate commits
        weblate.to.translations

        # copy the changes to the master branch
        cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"

        # compile translations
        build_msg BABEL 'compile translation catalogs into binary MO files'
        pybabel compile --statistics \
                -d "searx/translations"
        # git add/commit (no push)
        commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
        commit_message=$(echo -e "[translations] update\n${commit_body}")
        git add searx/translations
        git commit -m "${commit_message}"
    )
    exitcode=$?
    (   # make sure to always unlock weblate
        set -e
        pyenv.cmd wlc unlock
    )
    dump_return $exitcode
}

weblate.push.translations() {

    # Push *translation changes* from SearXNG (origin) to Weblate's counterpart
    # (weblate).

    # In branch master of SearXNG (origin) check for meaningful changes in
    # folder 'searx/translations', commit changes on branch 'translations' and
    # at least, pull updated branches on Weblate's counterpart (weblate).

    # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
    #    'translations' from remote 'weblate'.
    # 2. Stop if there is no meaningful change in the 'master' branch (origin),
    #    compared to the 'translations' branch (weblate), otherwise ...
    # 3. Update 'translations' branch of SearXNG (origin) with last additions
    #    from Weblate.
    # 5. Notify Weblate to pull updated 'master' & 'translations' branch.

    local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
          last_commit_message exitcode
    messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
    (   set -e
        pyenv.activate
        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
        weblate.translations.worktree

        # update messages.pot in the master branch
        build_msg BABEL 'extract messages from source files and generate POT file'
        pybabel extract -F babel.cfg \
                -o "${messages_pot}" \
                "searx/"

        # stop if there is no meaningful change in the master branch
        diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
                            git diff -- "searx/translations/messages.pot")
        if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
            build_msg BABEL 'no changes detected, exiting'
            return 42
        fi
        return 0
    )
    exitcode=$?
    if [ "$exitcode" -eq 42 ]; then
        return 0
    fi
    if [ "$exitcode" ]; then
       return $exitcode
    fi
    (
        set -e
        pyenv.activate

        # lock change on weblate
        # weblate may add commit(s) since the call to "weblate.translations.worktree".
        # this is not a problem because after this line, "weblate.to.translations"
        # calls again "weblate.translations.worktree" which calls "git pull"
        wlc lock

        # save messages.pot in the translations branch for later
        pushd "${TRANSLATIONS_WORKTREE}"
        git stash push
        popd

        # merge weblate commits into the translations branch
        weblate.to.translations

        # restore messages.pot in the translations branch
        pushd "${TRANSLATIONS_WORKTREE}"
        git stash pop
        popd

        # update messages.po files in the master branch
        build_msg BABEL 'update existing message catalogs from POT file'
        pybabel update -N \
            -i "${messages_pot}" \
            -d "${TRANSLATIONS_WORKTREE}/searx/translations"

        # git add/commit/push
        last_commit_hash=$(git log -n1  --pretty=format:'%h')
        last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
        last_commit_message="[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}"

        pushd "${TRANSLATIONS_WORKTREE}"
        git add searx/translations
        git commit -m "${last_commit_message}"
        git push
        popd

        # notify weblate to pull updated master & translations branch
        wlc pull
    )
    exitcode=$?
    (   # make sure to always unlock weblate
        set -e
        pyenv.activate
        wlc unlock
    )
    dump_return $exitcode
}

data.all() {
    (   set -e
        pyenv.activate
        data.languages
        data.useragents
        data.osm_keys_tags
        build_msg DATA "update searx/data/ahmia_blacklist.txt"
        python searx_extra/update/update_ahmia_blacklist.py
        build_msg DATA "update searx/data/wikidata_units.json"
        python searx_extra/update/update_wikidata_units.py
        build_msg DATA "update searx/data/currencies.json"
        python searx_extra/update/update_currencies.py
    )
}


data.languages() {
    (   set -e
        pyenv.activate
        build_msg ENGINES "fetch languages .."
        python searx_extra/update/update_languages.py
        build_msg ENGINES "update update searx/languages.py"
        build_msg DATA "update searx/data/engines_languages.json"
    )
    dump_return $?
}

data.useragents() {
    build_msg DATA "update searx/data/useragents.json"
    pyenv.cmd python searx_extra/update/update_firefox_version.py
    dump_return $?
}

data.osm_keys_tags() {
    build_msg DATA "update searx/data/osm_keys_tags.json"
    pyenv.cmd python searx_extra/update/update_osm_keys_tags.py
    dump_return $?
}

docs.prebuild() {
    build_msg DOCS "build ${DOCS_BUILD}/includes"
    (
        set -e
        [ "$VERBOSE" = "1" ] && set -x
        mkdir -p "${DOCS_BUILD}/includes"
        ./utils/searx.sh doc   | cat > "${DOCS_BUILD}/includes/searx.rst"
        ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
        ./utils/morty.sh doc   | cat > "${DOCS_BUILD}/includes/morty.rst"
    )
    dump_return $?
}

docker.push() {
    docker.build push
}

docker.buildx() {
    docker.build buildx
}

# shellcheck disable=SC2119
docker.build() {
    pyenv.install

    local SEARX_GIT_VERSION
    local VERSION_GITCOMMIT
    local GITHUB_USER
    local SEARX_IMAGE_NAME
    local BUILD

    build_msg DOCKER build
    # run installation in a subprocess and activate pyenv

    # See https://www.shellcheck.net/wiki/SC1001 and others ..
    # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
    (   set -e
        pyenv.activate

        # Check if it is a git repository
        if [ ! -d .git ]; then
	    die 1 "This is not Git repository"
        fi
        if [ ! -x "$(which git)" ]; then
	    die 1 "git is not installed"
        fi

        if ! git remote get-url origin 2> /dev/null; then
	    die 1 "there is no remote origin"
        fi

        # This is a git repository
        git update-index -q --refresh
        python -m searx.version freeze
        eval "$(python -m searx.version)"

        # Get the last git commit id
        VERSION_GITCOMMIT=$(echo "$VERSION_STRING" | cut -d- -f3)
        build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"

        # define the docker image name
        GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
        SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"

        BUILD="build"
        if [ "$1" = "buildx" ]; then
            # buildx includes the push option
            CACHE_TAG="${SEARX_IMAGE_NAME}:latest-build-cache"
            BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
            shift
        fi
        build_msg DOCKER "Build command: ${BUILD}"

        # build Docker image
        build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
        # shellcheck disable=SC2086
        docker $BUILD \
         --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
         --build-arg GIT_URL="${GIT_URL}" \
         --build-arg SEARX_GIT_VERSION="${VERSION_STRING}" \
         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
         --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
         --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
         --build-arg LABEL_VCS_URL="${GIT_URL}" \
         --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
         --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
         -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${VERSION_STRING}" .

        if [ "$1" = "push" ]; then
	        docker push "${SEARX_IMAGE_NAME}:latest"
	        docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
	    fi
    )
    dump_return $?
}

# shellcheck disable=SC2119
gecko.driver() {
    pyenv.install

    build_msg INSTALL "gecko.driver"
    # run installation in a subprocess and activate pyenv
    (   set -e
        pyenv.activate

        # TODO : check the current geckodriver version
        geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
        set +e
        if [ -z "$NOTFOUND" ]; then
            build_msg INSTALL "geckodriver already installed"
            return
        fi
        PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
        case "$PLATFORM" in
            "linux 32bit" | "linux2 32bit") ARCH="linux32";;
            "linux 64bit" | "linux2 64bit") ARCH="linux64";;
            "windows 32 bit") ARCH="win32";;
            "windows 64 bit") ARCH="win64";;
            "mac 64bit") ARCH="macos";;
        esac
        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";

        build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"

        FILE="$(mktemp)"
        wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
        rm -- "$FILE"
        chmod 755 -- "${PY_ENV_BIN}/geckodriver"
    )
    dump_return $?
}

node.env() {
    if ! required_commands npm; then
        info_msg "to install build tools use::"
        info_msg "   sudo -H ./utils/searx.sh install buildhost"
        die 1 "install needed build tools first"
    fi

    (   set -e

        build_msg INSTALL "searx/static/themes/oscar/package.json"
        npm --prefix searx/static/themes/oscar install

        build_msg INSTALL "searx/static/themes/simple/package.json"
        npm --prefix searx/static/themes/simple install
    )
    dump_return $?
}

node.clean() {
    if ! required_commands npm 2>/dev/null; then
        build_msg CLEAN "npm is not installed / ignore npm dependencies"
        return 0
    fi
    build_msg CLEAN "locally installed npm dependencies"
    (   set -e
        npm --prefix searx/static/themes/oscar run clean
        npm --prefix searx/static/themes/simple run clean
    )
    dump_return $?
}

pygments.less() {
    build_msg PYGMENTS "searx_extra/update/update_pygments.py"
    if ! pyenv.cmd python searx_extra/update/update_pygments.py; then
        build_msg PYGMENTS "building LESS files for pygments failed"
        return 1
    fi
    return 0
}

py.build() {
    build_msg BUILD "python package ${PYDIST}"
    pyenv.cmd python setup.py \
              sdist -d "${PYDIST}" \
              bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
}

py.clean() {
    build_msg CLEAN pyenv
    (   set -e
        pyenv.drop
        [ "$VERBOSE" = "1" ] && set -x
        rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
        find . -name '*.pyc' -exec rm -f {} +
        find . -name '*.pyo' -exec rm -f {} +
        find . -name __pycache__ -exec rm -rf {} +
    )
}

pyenv.check() {
    cat  <<EOF
import yaml
print('import yaml --> OK')
EOF
}

pyenv.install() {

    if ! pyenv.OK; then
        py.clean > /dev/null
    fi
    if pyenv.install.OK > /dev/null; then
        return 0
    fi

    (   set -e
        pyenv
        build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
        "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
        buildenv
    )
    local exit_val=$?
    if [ ! $exit_val -eq 0 ]; then
        die 42 "error while pip install (${PY_ENV_BIN})"
    fi
}

pyenv.uninstall() {
    build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
    pyenv.cmd python setup.py develop --uninstall 2>&1 \
        | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "

}

pypi.upload() {
    py.clean
    py.build
    # https://github.com/pypa/twine
    pyenv.cmd twine upload "${PYDIST}"/*
}

pypi.upload.test() {
    py.clean
    py.build
    pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
}

test.yamllint() {
    build_msg TEST "[yamllint] \$YAMLLINT_FILES"
    pyenv.cmd yamllint --format parsable "${YAMLLINT_FILES[@]}"
}

test.pylint() {
    # shellcheck disable=SC2086
    (   set -e
        build_msg TEST "[pylint] \$PYLINT_FILES"
        pyenv.activate
        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
            "${PYLINT_FILES[@]}"

        build_msg TEST "[pylint] searx/engines"
        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
            --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
            searx/engines

        build_msg TEST "[pylint] searx tests"
        python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
            --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
	    --ignore=searx/engines \
	    searx tests
    )
    dump_return $?
}

test.pep8() {
    build_msg TEST 'pycodestyle (formerly pep8)'
    local _exclude=""
    printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
    pyenv.cmd pycodestyle \
              --exclude="searx/static, searx/languages.py, $_exclude " \
              --max-line-length=120 \
              --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
              searx tests
    dump_return $?
}

test.unit() {
    build_msg TEST 'tests/unit'
    pyenv.cmd python -m nose2 -s tests/unit
    dump_return $?
}

test.coverage() {
    build_msg TEST 'unit test coverage'
    (   set -e
        pyenv.activate
        python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
        coverage report
        coverage html
    )
    dump_return $?
}

test.robot() {
    build_msg TEST 'robot'
    gecko.driver
    PYTHONPATH=. pyenv.cmd python -m tests.robot
    dump_return $?
}

test.clean() {
    build_msg CLEAN  "test stuff"
    rm -rf geckodriver.log .coverage coverage/
    dump_return $?
}

themes.all() {
    (   set -e
        pygments.less
        node.env
        themes.oscar
        themes.simple
    )
    dump_return $?
}

themes.oscar() {
    build_msg GRUNT "theme: oscar"
    npm --prefix searx/static/themes/oscar run build
    dump_return $?
}

themes.simple() {
    build_msg GRUNT "theme: simple"
    npm --prefix searx/static/themes/simple run build
    dump_return $?
}

PYLINT_FILES=()
while IFS= read -r line; do
   PYLINT_FILES+=("$line")
done <<< "$(pylint.FILES)"

# shellcheck disable=SC2119
main() {

    local _type
    local cmd="$1"; shift

    if [ "$cmd" == "" ]; then
        help
        err_msg "missing command"
        return 42
    fi

    case "$cmd" in
        --getenv) var="$1"; echo "${!var}";;
        --help) help;;
        --*)
            help
            err_msg "unknown option $cmd"
            return 42
            ;;
        *)
            _type="$(type -t "$cmd")"
            if [ "$_type" != 'function' ]; then
                err_msg "unknown command: $cmd / use --help"
                return 42
            else
                "$cmd" "$@"
            fi
            ;;
    esac
}

main "$@"