From b18880985c0b5948ebb1136135d8d05d292aae70 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 11:15:34 +0200 Subject: [PATCH 01/12] travis: make travis build more verbose Signed-off-by: Markus Heiser --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c3f7285c..59913af90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,15 +8,15 @@ addons: firefox: "latest" install: - - make install - - make gecko.driver - - make node.env + - make V=1 install + - make V=1 gecko.driver + - make V=1 node.env - local/py3/bin/pip install codecov script: - - make themes - - make test + - make V=1 themes + - make V=1 test after_success: - - make test.coverage + - make V=1 test.coverage - codecov stages: From 8c145ff2fed095edb79461c3be7ad76abc68db51 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 12:22:08 +0200 Subject: [PATCH 02/12] travis: build on os: linux, dist: bionic, use platform arch: arm64 Signed-off-by: Markus Heiser --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 59913af90..c79c5c4c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ +os: linux +dist: bionic +arch: arm64 language: python -sudo: false cache: - directories: - $HOME/.cache/pip From 9b48ae47ec837250a9a15e26f85ada33b7aa4f71 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 14:16:49 +0200 Subject: [PATCH 03/12] makefile.python: hardening - drop virtualenv, drop build support py2 From py3.3 on a subset of virtualenv is built-in, so you can run '-m venv' ot of the box. - replace: $(PY_ENV_BIN)/pip --> $(PY_ENV_BIN)/python -m pip - remove obsolete virtualenv-exe target and adjust VTENV_OPTS - remove obsolete msg-pip-exe target - print list of py launchers available from $(PY_ENV_BIN) to the log - fix hard coded ./local Signed-off-by: Markus Heiser --- .travis.yml | 1 - utils/makefile.python | 90 +++++++++++++------------------------------ 2 files changed, 26 insertions(+), 65 deletions(-) diff --git a/.travis.yml b/.travis.yml index c79c5c4c5..911ca98e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,6 @@ stages: jobs: include: - - python: "2.7" - python: "3.5" - python: "3.6" - stage: docker diff --git a/utils/makefile.python b/utils/makefile.python index 4aa9d6b49..699bef5e1 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -5,6 +5,7 @@ PYOBJECTS ?= SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH +export PY_ENV PYDIST PYBUILD # folder where the python distribution takes place PYDIST ?= ./py_dist @@ -12,6 +13,9 @@ PYDIST ?= ./py_dist PYBUILD ?= ./py_build # python version to use PY ?=3 +# $(PYTHON) points to the python interpreter from the OS! The python from the +# OS is needed e.g. to create a virtualenv. For tasks inside the virtualenv the +# interpeter from '$(PY_ENV_BIN)/python' is used. PYTHON ?= python$(PY) PIP ?= pip$(PY) PIP_INST ?= --user @@ -26,7 +30,7 @@ PYLINT_RC ?= .pylintrc TEST_FOLDER ?= ./tests TEST ?= . -VTENV_OPTS = "--no-site-packages" +VTENV_OPTS ?= PY_ENV = ./local/py$(PY) PY_ENV_BIN = $(PY_ENV)/bin PY_ENV_ACT = . $(PY_ENV_BIN)/activate @@ -37,20 +41,6 @@ ifeq ($(OS),Windows_NT) PY_ENV_ACT = $(PY_ENV_BIN)/activate endif -ifeq ($(PYTHON),python) - VIRTUALENV = virtualenv -else - VIRTUALENV = virtualenv --python=$(PYTHON) -endif - -ifeq ($(KBUILD_VERBOSE),1) - PIP_VERBOSE = - VIRTUALENV_VERBOSE = -else - PIP_VERBOSE = "-q" - VIRTUALENV_VERBOSE = "-q" -endif - python-help:: @echo 'makefile.python:' @echo ' pyenv | pyenv[un]install' @@ -59,7 +49,7 @@ python-help:: @echo ' pylint - run pylint *linting*' @echo ' pytest - run *tox* test on python objects' @echo ' pydebug - run tests within a PDB debug session' - @echo ' pybuild - build python packages' + @echo ' pybuild - build python packages ($(PYDIST) $(PYBUILD))' @echo ' pyclean - clean intermediate python objects' @echo ' targets using system users environment:' @echo ' py[un]install - [un]install python objects in editable mode' @@ -94,38 +84,6 @@ python-exe: @: endif -msg-pip-exe: - @echo "\n $(PIP) is required\n\n\ - Make sure you have updated pip installed, grab it from\n\ - https://pip.pypa.io or install it from your package\n\ - manager. On debian based OS these requirements are\n\ - installed by::\n\n\ - sudo -H apt-get install python$(PY)-pip\n" | $(FMT) - -ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1) -pip-exe: msg-pip-exe - $(error The '$(PIP)' command was not found) -else -pip-exe: - @: -endif - -PHONY += msg-virtualenv-exe virtualenv-exe -msg-virtualenv-exe: - @echo "\n virtualenv is required\n\n\ - Make sure you have an updated virtualenv installed, grab it from\n\ - https://virtualenv.pypa.io/en/stable/installation/ or install it\n\ - via pip by::\n\n\ - pip install --user https://github.com/pypa/virtualenv/tarball/master\n" | $(FMT) - -ifeq ($(shell which virtualenv >/dev/null 2>&1; echo $$?), 1) -virtualenv-exe: msg-virtualenv-exe - $(error The 'virtualenv' command was not found) -else -virtualenv-exe: - @: -endif - # ------------------------------------------------------------------------------ # commands # ------------------------------------------------------------------------------ @@ -136,9 +94,9 @@ quiet_cmd_pyinstall = INSTALL $2 # $2 path to folder with setup.py, this uses pip from pyenv (not OS!) quiet_cmd_pyenvinstall = PYENV install $2 - cmd_pyenvinstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) + cmd_pyenvinstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) -# Uninstall the package. Since pip does not uninstall the no longer needed +# Uninstall the package. Since pip does not uninstall the no longer needed # depencies (something like autoremove) the depencies remain. # $2 package name to uninstall, this uses pip from the OS. @@ -147,16 +105,19 @@ quiet_cmd_pyuninstall = UNINSTALL $2 # $2 path to folder with setup.py, this uses pip from pyenv (not OS!) quiet_cmd_pyenvuninstall = PYENV uninstall $2 - cmd_pyenvuninstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes $2 + cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2 # $2 path to folder where virtualenv take place +# creating a virtualenv uses $(PYTHON) from the OS! quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate cmd_virtualenv = \ - if [ ! -d "./$(PY_ENV)" ];then \ - $(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2; \ + if [ ! -d "$(PY_ENV)" ];then \ + echo "PYENV create virtualenv $2"; \ + $(PYTHON) -m venv $(VTENV_OPTS) $2; \ else \ echo "PYENV using virtualenv from $2"; \ - fi + fi; \ + echo "commands available at $(PY_ENV_BIN):"; ls $(PY_ENV_BIN) | $(FMT) ; # $2 path to lint quiet_cmd_pylint = LINT $@ @@ -192,14 +153,14 @@ quiet_cmd_pytest = TEST $@ # .. _installing: https://packaging.python.org/tutorials/installing-packages/ # quiet_cmd_pybuild = BUILD $@ - cmd_pybuild = $(PY_ENV_BIN)/$(PYTHON) setup.py \ + cmd_pybuild = $(PY_ENV_BIN)/python setup.py \ sdist -d $(PYDIST) \ bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST) quiet_cmd_pyclean = CLEAN $@ # remove 'build' folder since bdist_wheel does not care the --bdist-dir cmd_pyclean = \ - rm -rf $(PYDIST) $(PYBUILD) ./local ./.tox *.egg-info ;\ + rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info ;\ find . -name '*.pyc' -exec rm -f {} + ;\ find . -name '*.pyo' -exec rm -f {} + ;\ find . -name __pycache__ -exec rm -rf {} + @@ -230,15 +191,16 @@ PHONY += pyclean pyclean: $(call cmd,pyclean) -# to build *local* environment, python and virtualenv from the OS is needed! +# to build *local* environment, python from the OS is needed! pyenv: $(PY_ENV) -$(PY_ENV): virtualenv-exe python-exe +$(PY_ENV): python-exe $(call cmd,virtualenv,$(PY_ENV)) - @$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) -r requirements.txt + $(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel pip setuptools + $(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt PHONY += pylint-exe pylint-exe: $(PY_ENV) - @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install pylint + @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install pylint PHONY += pylint pylint: pylint-exe @@ -262,15 +224,15 @@ pydebug: $(PY_ENV) # install / uninstall python objects into virtualenv (PYENV) pyenv-install: $(PY_ENV) - @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e . + @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e . @echo "ACTIVATE $(call normpath,$(PY_ENV_ACT)) " pyenv-uninstall: $(PY_ENV) - @$(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes . + @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes . # runs python interpreter from ./local/py/bin/python pyenv-python: pyenv-install - cd ./local; ../$(PY_ENV_BIN)/python -i + $(PY_ENV_BIN)/python -i # With 'dependency_links=' setuptools supports dependencies on packages hosted # on other reposetories then PyPi, see "Packages Not On PyPI" [1]. The big @@ -284,7 +246,7 @@ pyenv-python: pyenv-install # https://github.com/pypa/twine PHONY += upload-pypi -upload-pypi: pyclean pybuild +upload-pypi: pyclean pyenvinstall pybuild @$(PY_ENV_BIN)/twine upload $(PYDIST)/* .PHONY: $(PHONY) From 9e32fb27e71ca0ae32a0eead712a0504c9d2d951 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 14:49:10 +0200 Subject: [PATCH 04/12] [fix] travis: Permission denied: '/home/travis/.cache/pip/...' Fix "Permission denied: '/home/travis/.cache/" errors from travis build [1]. Change owner https://docs.travis-ci.com/user/caching/#caches-and-read-permissions and remove old log in the *before_caching* phase: https://docs.travis-ci.com/user/caching/#before_cache-phase About pip cache see: https://docs.travis-ci.com/user/caching/#pip-cache Signed-off-by: Markus Heiser [1] https://travis-ci.org/github/asciimoo/searx/jobs/669664131#L220 --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 911ca98e1..94303773d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,9 @@ language: python cache: - directories: - $HOME/.cache/pip - +before_cache: + - sudo chown -R travis:travis $HOME/.cache/pip + - rm -f $HOME/.cache/pip/log/debug.log addons: firefox: "latest" From eb87b07d519592b8863a3aeb39eb0a758083f5cc Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 15:06:48 +0200 Subject: [PATCH 05/12] travis: disable cache Signed-off-by: Markus Heiser --- .travis.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94303773d..7d896fffe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,7 @@ os: linux dist: bionic arch: arm64 language: python -cache: - - directories: - - $HOME/.cache/pip -before_cache: - - sudo chown -R travis:travis $HOME/.cache/pip - - rm -f $HOME/.cache/pip/log/debug.log +cache: false addons: firefox: "latest" From adc3a855e6eb4626014c0459a13e868e65bcfd74 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 15:22:24 +0200 Subject: [PATCH 06/12] Revert "travis: disable cache" This reverts commit eb87b07d519592b8863a3aeb39eb0a758083f5cc. --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d896fffe..94303773d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,12 @@ os: linux dist: bionic arch: arm64 language: python -cache: false +cache: + - directories: + - $HOME/.cache/pip +before_cache: + - sudo chown -R travis:travis $HOME/.cache/pip + - rm -f $HOME/.cache/pip/log/debug.log addons: firefox: "latest" From e340bb5bd18c805bbb8381ec8c6052bf95830bc2 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 15:27:01 +0200 Subject: [PATCH 07/12] travis: remove platform option arm64 [1] https://github.com/asciimoo/searx/pull/1910#issuecomment-607244871 Signed-off-by: Markus Heiser --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 94303773d..c9847d632 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ os: linux dist: bionic -arch: arm64 language: python cache: - directories: From 565493b57d3dbef94f4c298b7395e26fc24bb9bd Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 15:45:01 +0200 Subject: [PATCH 08/12] travis: use Makefile target travis.codecov to install codecov fix travis build error [1]:: The command "local/py3/bin/pip install codecov" failed and exited with 127 Use the correct pip (python environment) from build environment:: $(PY_ENV_BIN)/python -m pip [1] https://travis-ci.org/github/asciimoo/searx/jobs/669701405#L590 Signed-off-by: Markus Heiser --- .travis.yml | 2 +- Makefile | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c9847d632..e9f727a32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ install: - make V=1 install - make V=1 gecko.driver - make V=1 node.env - - local/py3/bin/pip install codecov + - make V=1 travis.codecov script: - make V=1 themes - make V=1 test diff --git a/Makefile b/Makefile index 65f79ba2c..7d5282802 100644 --- a/Makefile +++ b/Makefile @@ -220,4 +220,11 @@ test.clean: @echo "CLEAN intermediate test stuff" $(Q)rm -rf geckodriver.log .coverage coverage/ + +# travis +# ------ + +travis.codecov: + $(Q)$(PY_ENV_BIN)/python -m pip install codecov + .PHONY: $(PHONY) From 31437d04e82ec9cde4209d881f6635ac39d92ac4 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 16:07:59 +0200 Subject: [PATCH 09/12] makefile.python: hardening - start pylint by module - replace: $(PY_ENV_BIN)/pylint --> $(PY_ENV_BIN)/python -m pylint Signed-off-by: Markus Heiser --- utils/makefile.python | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/makefile.python b/utils/makefile.python index 699bef5e1..aaf21c115 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -121,10 +121,10 @@ quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate # $2 path to lint quiet_cmd_pylint = LINT $@ - cmd_pylint = $(PY_ENV_BIN)/pylint --rcfile $(PYLINT_RC) $2 + cmd_pylint = $(PY_ENV_BIN)/python -m pylint --rcfile $(PYLINT_RC) $2 quiet_cmd_pytest = TEST $@ - cmd_pytest = $(PY_ENV_BIN)/tox -vv + cmd_pytest = $(PY_ENV_BIN)/python -m tox -vv # setuptools, pip, easy_install its a mess full of cracks, a documentation hell # and broken by design ... all sucks, I really, really hate all this ... aaargh! From 09a4062505a8d79451374139bab525457eeb28cd Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 17:21:26 +0200 Subject: [PATCH 10/12] makefile.python: partial revert off 9b48ae47, adds py2 support back revert "makefile.python: ... drop build support py2" to get back Py2 support. TPy2 support need virtualenv installed by the OS. BTW: log environment and python version in travis's install phase Signed-off-by: Markus Heiser --- .travis.yml | 3 +++ utils/makefile.python | 25 ++++++++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index e9f727a32..3c862fac6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ addons: firefox: "latest" install: + - env; which python; python --version - make V=1 install - make V=1 gecko.driver - make V=1 node.env @@ -29,6 +30,8 @@ stages: jobs: include: + - python: "2.7" + env: PY=2 - python: "3.5" - python: "3.6" - stage: docker diff --git a/utils/makefile.python b/utils/makefile.python index aaf21c115..590bbdb46 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -30,7 +30,7 @@ PYLINT_RC ?= .pylintrc TEST_FOLDER ?= ./tests TEST ?= . -VTENV_OPTS ?= +VTENV_OPTS = "--no-site-packages" PY_ENV = ./local/py$(PY) PY_ENV_BIN = $(PY_ENV)/bin PY_ENV_ACT = . $(PY_ENV_BIN)/activate @@ -41,6 +41,20 @@ ifeq ($(OS),Windows_NT) PY_ENV_ACT = $(PY_ENV_BIN)/activate endif +ifeq ($(PYTHON),python) + VIRTUALENV = virtualenv +else + VIRTUALENV = virtualenv --python=$(PYTHON) +endif + +ifeq ($(KBUILD_VERBOSE),1) + PIP_VERBOSE = + VIRTUALENV_VERBOSE = +else + PIP_VERBOSE = "-q" + VIRTUALENV_VERBOSE = "-q" +endif + python-help:: @echo 'makefile.python:' @echo ' pyenv | pyenv[un]install' @@ -108,16 +122,13 @@ quiet_cmd_pyenvuninstall = PYENV uninstall $2 cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2 # $2 path to folder where virtualenv take place -# creating a virtualenv uses $(PYTHON) from the OS! quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate cmd_virtualenv = \ - if [ ! -d "$(PY_ENV)" ];then \ - echo "PYENV create virtualenv $2"; \ - $(PYTHON) -m venv $(VTENV_OPTS) $2; \ + if [ ! -d "./$(PY_ENV)" ];then \ + $(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2; \ else \ echo "PYENV using virtualenv from $2"; \ - fi; \ - echo "commands available at $(PY_ENV_BIN):"; ls $(PY_ENV_BIN) | $(FMT) ; + fi # $2 path to lint quiet_cmd_pylint = LINT $@ From fc89c698bd81a582b5ea7095de32a12d54a40268 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Wed, 1 Apr 2020 19:20:52 +0200 Subject: [PATCH 11/12] make test.pylint: do not run pylint checks in py2 Signed-off-by: Markus Heiser --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 7d5282802..abd0eddb8 100644 --- a/Makefile +++ b/Makefile @@ -186,12 +186,17 @@ PHONY += test test.pylint test.pep8 test.unit test.coverage test.robot test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot +ifeq ($(PY),2) +test.pylint: + @echo "LINT skip liniting py2" +else # TODO: balance linting with pylint test.pylint: pyenvinstall $(call cmd,pylint,\ searx/preferences.py \ searx/testing.py \ ) +endif # ignored rules: # E402 module level import not at top of file From 41f002b0ea3e82f9a2f7dd7b75dbbbf77b7986c0 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Thu, 2 Apr 2020 18:00:53 +0200 Subject: [PATCH 12/12] travis: remove before_cache handle BTW: separate env output in the travis log Signed-off-by: Markus Heiser --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c862fac6..76a20b00d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,12 @@ language: python cache: - directories: - $HOME/.cache/pip -before_cache: - - sudo chown -R travis:travis $HOME/.cache/pip - - rm -f $HOME/.cache/pip/log/debug.log addons: firefox: "latest" install: - - env; which python; python --version + - env + - which python; python --version - make V=1 install - make V=1 gecko.driver - make V=1 node.env