From d0cebd6e7787c086fce138b177d4db48eb8e1c96 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 13 Aug 2017 09:33:12 +0200 Subject: [PATCH 1/4] [fix] fix selenium tests --- .travis.yml | 5 ++++- manage.sh | 2 +- requirements-dev.txt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1eb6573f2..8ddbc7583 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,13 @@ before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" - npm install less less-plugin-clean-css grunt-cli + - export PATH=`pwd`/node_modules/.bin:$PATH + - ls -l $VIRTUAL_ENV + - ls -l $VIRTUAL_ENV/bin - ( cd searx/static/themes/oscar;npm install; cd - ) - ( cd searx/static/themes/simple;npm install; cd - ) - mkdir -p ~/drivers; export PATH=~/drivers:$PATH; - - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.14.0/geckodriver-v0.14.0-linux64.tar.gz"; + - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.18.0/geckodriver-v0.18.0-linux64.tar.gz"; - FILE=`mktemp`; wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C ~/drivers -f $FILE geckodriver; rm $FILE; chmod 777 ~/drivers/geckodriver; install: - ./manage.sh update_dev_packages diff --git a/manage.sh b/manage.sh index 681f3a975..8408e5678 100755 --- a/manage.sh +++ b/manage.sh @@ -22,7 +22,7 @@ check_geckodriver() { if [ -z $NOTFOUND ]; then return fi - GECKODRIVER_VERSION="v0.14.0" + GECKODRIVER_VERSION="v0.18.0" PLATFORM=`python -c "import platform; print platform.system().lower(), platform.architecture()[0]"` case $PLATFORM in "linux 32bit" | "linux2 32bit") ARCH="linux32";; diff --git a/requirements-dev.txt b/requirements-dev.txt index 691a1e7ba..8c2b72305 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,4 +7,4 @@ splinter==0.7.5 transifex-client==0.12.2 unittest2==1.1.0 zope.testrunner==4.5.1 -selenium==3.0.1 +selenium==3.5.0 From 57dc6b625f85a39f722727b2fd107a64637a8bb9 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 13 Aug 2017 12:17:26 +0200 Subject: [PATCH 2/4] [fix] travis build : various fix about grunt builds --- .travis.yml | 5 +-- manage.sh | 47 ++++++++++++++++--------- searx/static/themes/oscar/gruntfile.js | 4 +++ searx/static/themes/simple/gruntfile.js | 1 + 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8ddbc7583..aeab28885 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,7 @@ before_install: - "sh -e /etc/init.d/xvfb start" - npm install less less-plugin-clean-css grunt-cli - export PATH=`pwd`/node_modules/.bin:$PATH - - ls -l $VIRTUAL_ENV - - ls -l $VIRTUAL_ENV/bin - - ( cd searx/static/themes/oscar;npm install; cd - ) - - ( cd searx/static/themes/simple;npm install; cd - ) + - ./manage.sh grunt_packages - mkdir -p ~/drivers; export PATH=~/drivers:$PATH; - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.18.0/geckodriver-v0.18.0-linux64.tar.gz"; - FILE=`mktemp`; wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C ~/drivers -f $FILE geckodriver; rm $FILE; chmod 777 ~/drivers/geckodriver; diff --git a/manage.sh b/manage.sh index 8408e5678..8a98f2bfe 100755 --- a/manage.sh +++ b/manage.sh @@ -5,6 +5,8 @@ PYTHONPATH=$BASE_DIR SEARX_DIR="$BASE_DIR/searx" ACTION=$1 +cd $BASE_DIR + update_packages() { pip install -r "$BASE_DIR/requirements.txt" } @@ -14,7 +16,7 @@ update_dev_packages() { pip install -r "$BASE_DIR/requirements-dev.txt" } -check_geckodriver() { +install_geckodriver() { echo '[!] Checking geckodriver' set -e geckodriver -V 2>1 > /dev/null || NOTFOUND=1 @@ -73,37 +75,47 @@ tests() { set -e pep8_check unit_tests - check_geckodriver + install_geckodriver robot_tests set +e } build_style() { - # lessc -x "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2" lessc --clean-css="--s1 --advanced --compatibility=ie9" "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2" } styles() { echo '[!] Building styles' - build_style themes/legacy/less/style.less themes/legacy/css/style.css - build_style themes/legacy/less/style-rtl.less themes/legacy/css/style-rtl.css - build_style themes/courgette/less/style.less themes/courgette/css/style.css - build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css - build_style less/bootstrap/bootstrap.less css/bootstrap.min.css - build_style themes/oscar/less/pointhi/oscar.less themes/oscar/css/pointhi.min.css - build_style themes/oscar/less/logicodev/oscar.less themes/oscar/css/logicodev.min.css - build_style themes/pix-art/less/style.less themes/pix-art/css/style.css - build_style themes/simple/less/style.less themes/simple/css/searx.min.css - build_style themes/simple/less/style-rtl.less themes/simple/css/searx-rtl.min.css + build_style themes/legacy/less/style.less themes/legacy/css/style.css + build_style themes/legacy/less/style-rtl.less themes/legacy/css/style-rtl.css + build_style themes/courgette/less/style.less themes/courgette/css/style.css + build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css + build_style less/bootstrap/bootstrap.less css/bootstrap.min.css + build_style themes/oscar/less/pointhi/oscar.less themes/oscar/css/pointhi.min.css + build_style themes/oscar/less/logicodev/oscar.less themes/oscar/css/logicodev.min.css + build_style themes/pix-art/less/style.less themes/pix-art/css/style.css + build_style themes/simple/less/style.less themes/simple/css/searx.min.css + build_style themes/simple/less/style-rtl.less themes/simple/css/searx-rtl.min.css +} + +grunt_packages() { + echo '[!] Grunt packages: install dependencies' + cd $BASE_DIR/searx/static/themes/oscar + npm install + + cd $BASE_DIR/searx/static/themes/simple + npm install } grunt_build() { - grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js" - grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js" + echo '[!] Grunt build : oscar theme' + grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js" + echo '[!] Grunt build : simple theme' + grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js" } locales() { - pybabel compile -d "$SEARX_DIR/translations" + pybabel compile -d "$SEARX_DIR/translations" } help() { @@ -112,6 +124,7 @@ help() { Commands ======== + grunt_packages - Download & install dependencies grunt_build - Build js files help - This text locales - Compile locales @@ -123,7 +136,7 @@ Commands unit_tests - Run unit tests update_dev_packages - Check & update development and production dependency changes update_packages - Check & update dependency changes - check_geckodriver - Check & download geckodriver (required for robot_tests) + install_geckodriver - Download & install geckodriver if not already installed (required for robot_tests) " } diff --git a/searx/static/themes/oscar/gruntfile.js b/searx/static/themes/oscar/gruntfile.js index 59d1b6d96..625f5164d 100644 --- a/searx/static/themes/oscar/gruntfile.js +++ b/searx/static/themes/oscar/gruntfile.js @@ -24,6 +24,7 @@ module.exports = function(grunt) { jshint: { files: ['gruntfile.js', 'js/searx_src/*.js'], options: { + reporterOutput: "", // options here to override JSHint defaults globals: { jQuery: true, @@ -51,6 +52,8 @@ module.exports = function(grunt) { files: {"css/pointhi.min.css": "less/pointhi/oscar.less", "css/logicodev.min.css": "less/logicodev/oscar.less"} }, + /* + // built with ./manage.sh styles bootstrap: { options: { paths: ["less/bootstrap"], @@ -58,6 +61,7 @@ module.exports = function(grunt) { }, files: {"css/bootstrap.min.css": "less/bootstrap/bootstrap.less"} }, + */ }, watch: { scripts: { diff --git a/searx/static/themes/simple/gruntfile.js b/searx/static/themes/simple/gruntfile.js index 288ab6633..1a3e00d3d 100644 --- a/searx/static/themes/simple/gruntfile.js +++ b/searx/static/themes/simple/gruntfile.js @@ -34,6 +34,7 @@ module.exports = function(grunt) { jshint: { files: ['js/searx_src/*.js'], options: { + reporterOutput: "", proto: true, // options here to override JSHint defaults globals: { From d87eb4da6194975eeacd99f8ec197154497ba8bd Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 13 Aug 2017 13:45:02 +0200 Subject: [PATCH 3/4] [mod] travis build : travis uses ./manage.sh to install the geckodriver ./manage.sh don't build css that are built by ./manage.sh grunt_build --- .travis.yml | 7 +++---- manage.sh | 47 +++++++++++++++++++++++++++++------------------ 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index aeab28885..1da7974be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,10 @@ before_install: - "sh -e /etc/init.d/xvfb start" - npm install less less-plugin-clean-css grunt-cli - export PATH=`pwd`/node_modules/.bin:$PATH - - ./manage.sh grunt_packages - - mkdir -p ~/drivers; export PATH=~/drivers:$PATH; - - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v0.18.0/geckodriver-v0.18.0-linux64.tar.gz"; - - FILE=`mktemp`; wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C ~/drivers -f $FILE geckodriver; rm $FILE; chmod 777 ~/drivers/geckodriver; + - ./manage.sh install_geckodriver ~/drivers + - export PATH=~/drivers:$PATH install: + - ./manage.sh npm_packages - ./manage.sh update_dev_packages - pip install coveralls script: diff --git a/manage.sh b/manage.sh index 8a98f2bfe..28a208b7d 100755 --- a/manage.sh +++ b/manage.sh @@ -25,7 +25,7 @@ install_geckodriver() { return fi GECKODRIVER_VERSION="v0.18.0" - PLATFORM=`python -c "import platform; print platform.system().lower(), platform.architecture()[0]"` + PLATFORM=`python -c "import six; import platform; six.print_(platform.system().lower(), platform.architecture()[0])"` case $PLATFORM in "linux 32bit" | "linux2 32bit") ARCH="linux32";; "linux 64bit" | "linux2 64bit") ARCH="linux64";; @@ -34,16 +34,25 @@ install_geckodriver() { "mac 64bit") ARCH="macos";; esac GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; - if [ -z "$VIRTUAL_ENV" ]; then - echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n $GECKODRIVER_URL" - exit + + if [ -z $1 ]; then + if [ -z "$VIRTUAL_ENV" ]; then + echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n $GECKODRIVER_URL" + exit + else + GECKODRIVER_DIR=$VIRTUAL_ENV/bin + fi else - echo "Installing $VIRTUAL_ENV from\n $GECKODRIVER_URL" - FILE=`mktemp` - wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C $VIRTUAL_ENV/bin/ -f $FILE geckodriver - rm $FILE - chmod 777 $VIRTUAL_ENV/bin/geckodriver + GECKODRIVER_DIR=$1 + mkdir -p $GECKODRIVER_DIR fi + + echo "Installing $GECKODRIVER_DIR from\n $GECKODRIVER_URL" + + FILE=`mktemp` + wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C $GECKODRIVER_DIR -f $FILE geckodriver + rm $FILE + chmod 777 $GECKODRIVER_DIR/geckodriver } pep8_check() { @@ -91,18 +100,20 @@ styles() { build_style themes/courgette/less/style.less themes/courgette/css/style.css build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css build_style less/bootstrap/bootstrap.less css/bootstrap.min.css - build_style themes/oscar/less/pointhi/oscar.less themes/oscar/css/pointhi.min.css - build_style themes/oscar/less/logicodev/oscar.less themes/oscar/css/logicodev.min.css build_style themes/pix-art/less/style.less themes/pix-art/css/style.css - build_style themes/simple/less/style.less themes/simple/css/searx.min.css - build_style themes/simple/less/style-rtl.less themes/simple/css/searx-rtl.min.css + # built using grunt + #build_style themes/oscar/less/pointhi/oscar.less themes/oscar/css/pointhi.min.css + #build_style themes/oscar/less/logicodev/oscar.less themes/oscar/css/logicodev.min.css + #build_style themes/simple/less/style.less themes/simple/css/searx.min.css + #build_style themes/simple/less/style-rtl.less themes/simple/css/searx-rtl.min.css } -grunt_packages() { - echo '[!] Grunt packages: install dependencies' +npm_packages() { + echo '[!] install NPM packages for oscar theme' cd $BASE_DIR/searx/static/themes/oscar npm install - + + echo '[!] install NPM packages for simple theme' cd $BASE_DIR/searx/static/themes/simple npm install } @@ -124,7 +135,7 @@ help() { Commands ======== - grunt_packages - Download & install dependencies + npm_packages - Download & install dependencies grunt_build - Build js files help - This text locales - Compile locales @@ -142,4 +153,4 @@ Commands [ "$(command -V "$ACTION" | grep ' function$')" = "" ] \ && help "action not found" \ - || $ACTION + || $ACTION $2 From 1a814369fda1c813ddeed5f4052b15563b11a403 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 16 Aug 2017 18:14:07 +0200 Subject: [PATCH 4/4] [fix] manage.sh: add quotes arround parameters dealing with directories or files. --- manage.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/manage.sh b/manage.sh index 28a208b7d..732c099a3 100755 --- a/manage.sh +++ b/manage.sh @@ -5,7 +5,7 @@ PYTHONPATH=$BASE_DIR SEARX_DIR="$BASE_DIR/searx" ACTION=$1 -cd $BASE_DIR +cd "$BASE_DIR" update_packages() { pip install -r "$BASE_DIR/requirements.txt" @@ -18,6 +18,7 @@ update_dev_packages() { install_geckodriver() { echo '[!] Checking geckodriver' + # TODO : check the current geckodriver version set -e geckodriver -V 2>1 > /dev/null || NOTFOUND=1 set +e @@ -35,24 +36,24 @@ install_geckodriver() { esac GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; - if [ -z $1 ]; then + if [ -z "$1" ]; then if [ -z "$VIRTUAL_ENV" ]; then echo "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n $GECKODRIVER_URL" exit else - GECKODRIVER_DIR=$VIRTUAL_ENV/bin + GECKODRIVER_DIR="$VIRTUAL_ENV/bin" fi else - GECKODRIVER_DIR=$1 - mkdir -p $GECKODRIVER_DIR + GECKODRIVER_DIR="$1" + mkdir -p "$GECKODRIVER_DIR" fi - echo "Installing $GECKODRIVER_DIR from\n $GECKODRIVER_URL" + echo "Installing $GECKODRIVER_DIR/geckodriver from\n $GECKODRIVER_URL" FILE=`mktemp` - wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C $GECKODRIVER_DIR -f $FILE geckodriver + wget "$GECKODRIVER_URL" -qO $FILE && tar xz -C "$GECKODRIVER_DIR" -f $FILE geckodriver rm $FILE - chmod 777 $GECKODRIVER_DIR/geckodriver + chmod 777 "$GECKODRIVER_DIR/geckodriver" } pep8_check() { @@ -153,4 +154,4 @@ Commands [ "$(command -V "$ACTION" | grep ' function$')" = "" ] \ && help "action not found" \ - || $ACTION $2 + || $ACTION "$2"