diff --git a/Makefile b/Makefile index ac63842e1..d46c96a49 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,12 @@ # -*- coding: utf-8; mode: makefile-gmake -*- .DEFAULT_GOAL=help include ./.config.mk +include utils/makefile.include PYOBJECTS = searx DOC = docs PY_SETUP_EXTRAS ?= \[test\] -PYDIST=./dist/py -PYBUILD=./build/py - -include utils/makefile.include include utils/makefile.python include utils/makefile.sphinx @@ -32,9 +29,9 @@ help: @echo ' GIT_URL = $(GIT_URL)' @echo ' DOCS_URL = $(DOCS_URL)' @echo '' - @$(MAKE) -s -f utils/makefile.include make-help + @$(MAKE) -e -s -f utils/makefile.include make-help @echo '' - @$(MAKE) -s -f utils/makefile.python python-help + @$(MAKE) -e -s -f utils/makefile.python python-help PHONY += install install: pyenvinstall @@ -43,7 +40,7 @@ PHONY += uninstall uninstall: pyenvuninstall PHONY += clean -clean: pyclean +clean: pyclean docs-clean $(call cmd,common_clean) PHONY += run diff --git a/utils/lxc-searx.env b/utils/lxc-searx.env index b088ca792..a5ef5712c 100644 --- a/utils/lxc-searx.env +++ b/utils/lxc-searx.env @@ -32,6 +32,7 @@ lxc_set_suite_env() { export MORTY_LISTEN="0.0.0.0:3000" } +lxc_suite_install_info="suite includes searx, morty & filtron" lxc_suite_install() { ( lxc_set_suite_env @@ -46,6 +47,16 @@ lxc_suite_install() { ) } +lxc_suite_prepare_buildhost() { + ( + lxc_set_suite_env + export FORCE_TIMEOUT=0 + "${LXC_REPO_ROOT}/utils/searx.sh" install buildhost + rst_title "buildhost installation finished ($(hostname))" part + echo + ) +} + lxc_suite_info() { ( lxc_set_suite_env diff --git a/utils/lxc.sh b/utils/lxc.sh index 3c4d2016e..64805272e 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -104,7 +104,8 @@ cmd -- run command '...' in all containers of the LXC suite :: run command '...' in container install - :suite: install LXC suite, includes morty & filtron + :suite: install LXC suite; ${lxc_suite_install_info} + :buildhost: prepare LXC; buildhost EOF usage_images @@ -224,12 +225,7 @@ main() { sudo_or_exit shift case $1 in - --) - shift - for name in "${CONTAINERS[@]}"; do - lxc_exec_cmd "${name}" "$@" - done - ;; + --) shift; lxc_exec "$@" ;; ${LXC_HOST_PREFIX}-*) ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42 local name=$1 @@ -242,13 +238,15 @@ main() { install) sudo_or_exit case $2 in - suite) install_suite ;; + suite) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install suite;; + buildhost) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install buildhost;; *) usage "$_usage"; exit 42 ;; esac ;; __install) case $2 in suite) lxc_suite_install ;; + buildhost) lxc_suite_prepare_buildhost ;; esac ;; doc) @@ -263,7 +261,6 @@ main() { build_instances() { rst_title "Build LXC instances" - echo lxc_copy_images_localy echo rst_title "build containers" section @@ -360,18 +357,6 @@ show_suite(){ done } -install_suite() { - for i in "${CONTAINERS[@]}"; do - if ! lxc_exists "$i"; then - warn_msg "container ${_BBlue}$i${_creset} does not yet exists" - else - info_msg "[${_BBlue}${i}${_creset}] ${_BGreen}${LXC_REPO_ROOT}/utils/lxc.sh install suite${_creset}" - lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __install suite \ - | prefix_stdout "[${_BBlue}${i}${_creset}] " - fi - done -} - lxc_cmd() { for i in "${CONTAINERS[@]}"; do if ! lxc_exists "$i"; then @@ -389,14 +374,23 @@ lxc_exec_cmd() { shift exit_val= info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" - lxc exec --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" + lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" exit_val=$? if [[ $exit_val -ne 0 ]]; then warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" else info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" fi - echo +} + +lxc_exec() { + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done } lxc_init_containers() { @@ -449,7 +443,17 @@ lxc_boilerplate_containers() { boilerplate_script="${image_name}_boilerplate" boilerplate_script="${!boilerplate_script}" - info_msg "[${_BBlue}${container_name}${_creset}] install boilerplate" + info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + cat </dev/null; then sleep 5 # guest needs some time to come up and get an IP fi diff --git a/utils/makefile.include b/utils/makefile.include index 716889c02..933d2b57a 100644 --- a/utils/makefile.include +++ b/utils/makefile.include @@ -1,12 +1,23 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +ifeq (,$(wildcard /.lxcenv.mk)) +PHONY += lxc-activate +lxc-activate: + @$(MAKE) -s -f /share/searx/utils/makefile.lxc lxc-activate +else + include /.lxcenv.mk +endif + +ifeq (,$(wildcard /.lxcenv.mk)) make-help: +else +make-help: lxc-help +endif @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' @echo ' make V=2 [targets] 2 => give reason for rebuild of target' quiet_cmd_common_clean = CLEAN $@ cmd_common_clean = \ - rm -rf tests/build ;\ find . -name '*.orig' -exec rm -f {} + ;\ find . -name '*.rej' -exec rm -f {} + ;\ find . -name '*~' -exec rm -f {} + ;\ @@ -126,3 +137,4 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(ech # printing commands cmd = @$(echo-cmd) $(cmd_$(1)) +.PHONY: $(PHONY) diff --git a/utils/makefile.lxc b/utils/makefile.lxc new file mode 100644 index 000000000..8b79994ee --- /dev/null +++ b/utils/makefile.lxc @@ -0,0 +1,29 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- +# +# LXC environment +# =============== +# +# To activate/deactivate LXC makefile environment in a container, set/unset link +# from root '/.lxcenv.mk' to *this* file:: +# +# sudo make ./utils/makefile.lxc lxc-activate +# sudo make ./utils/makefile.lxc lxc-deactivate + +LXC_ENV_FOLDER=/lxc-$(shell hostname) + +lxc-help:: + @echo 'LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)' + +# If not activated, serve target 'lxc-activate' .. +ifeq (,$(wildcard /.lxcenv.mk)) +PHONY += lxc-activate +lxc-activate: + ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk" +else +# .. and if activated, serve target 'lxc-deactivate'. +PHONY += lxc-deactivate +lxc-deactivate: + rm /.lxcenv.mk +endif + +.PHONY: $(PHONY) diff --git a/utils/makefile.python b/utils/makefile.python index f7e4315c1..bd7d0d14e 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -5,11 +5,12 @@ PYOBJECTS ?= SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH +export PY_ENV PYDIST PYBUILD # folder where the python distribution takes place -PYDIST ?= ./py_dist +PYDIST = ./dist$(LXC_ENV_FOLDER) # folder where the python intermediate build files take place -PYBUILD ?= ./py_build +PYBUILD = ./build$(LXC_ENV_FOLDER) # python version to use PY ?=3 PYTHON ?= python$(PY) @@ -27,7 +28,7 @@ TEST_FOLDER ?= ./tests TEST ?= . VTENV_OPTS ?= -PY_ENV = ./local/py$(PY) +PY_ENV = ./local$(LXC_ENV_FOLDER)/py$(PY) PY_ENV_BIN = $(PY_ENV)/bin PY_ENV_ACT = . $(PY_ENV_BIN)/activate @@ -45,7 +46,7 @@ python-help:: @echo ' pylint - run pylint *linting*' @echo ' pytest - run *tox* test on python objects' @echo ' pydebug - run tests within a PDB debug session' - @echo ' pybuild - build python packages' + @echo ' pybuild - build python packages ($(PYDIST) $(PYBUILD))' @echo ' pyclean - clean intermediate python objects' @echo ' targets using system users environment:' @echo ' py[un]install - [un]install python objects in editable mode' @@ -169,7 +170,7 @@ quiet_cmd_pybuild = BUILD $@ quiet_cmd_pyclean = CLEAN $@ # remove 'build' folder since bdist_wheel does not care the --bdist-dir cmd_pyclean = \ - rm -rf $(PYDIST) $(PYBUILD) ./local ./.tox *.egg-info ;\ + rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info ;\ find . -name '*.pyc' -exec rm -f {} + ;\ find . -name '*.pyo' -exec rm -f {} + ;\ find . -name __pycache__ -exec rm -rf {} + @@ -241,7 +242,7 @@ pyenv-uninstall: $(PY_ENV) # runs python interpreter from ./local/py/bin/python pyenv-python: pyenv-install - cd ./local; ../$(PY_ENV_BIN)/python -i + $(PY_ENV_BIN)/python -i # With 'dependency_links=' setuptools supports dependencies on packages hosted # on other reposetories then PyPi, see "Packages Not On PyPI" [1]. The big diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx index 2c1922fc9..4926bb53a 100644 --- a/utils/makefile.sphinx +++ b/utils/makefile.sphinx @@ -1,17 +1,19 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST + # You can set these variables from the command line. SPHINXOPTS ?= SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build SPHINX_CONF ?= conf.py -DOCS_FOLDER ?= docs -DOCS_BUILD ?= build/docs -DOCS_DIST ?= dist/docs +DOCS_FOLDER = docs$(LXC_ENV_FOLDER) +DOCS_BUILD = build/docs$(LXC_ENV_FOLDER) +DOCS_DIST = dist/docs$(LXC_ENV_FOLDER) GH_PAGES ?= gh-pages -BOOKS_FOLDER ?= docs -BOOKS_DIST ?= dist/books +BOOKS_FOLDER = docs$(LXC_ENV_FOLDER) +BOOKS_DIST = dist/books$(LXC_ENV_FOLDER) ifeq ($(KBUILD_VERBOSE),1) SPHINX_VERBOSE = "-v"