From b9c73fb69784726608d1a2d72b7a17f777d30b9d Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 2 Sep 2021 16:01:34 +0200 Subject: [PATCH 1/4] [mod] move searx/testing.py to the tests directory move robot tests to tests.robot manage calls "python -m tests.robot" --- manage | 2 +- tests/__init__.py | 42 +++++++++++ tests/robot/__init__.py | 76 -------------------- searx/testing.py => tests/robot/__main__.py | 80 +++++---------------- {searx => tests/robot}/settings_robot.yml | 0 tests/robot/test_webapp.py | 78 ++++++++++++++++++++ tests/unit/engines/test_command.py | 2 +- tests/unit/engines/test_xpath.py | 2 +- tests/unit/network/test_network.py | 2 +- tests/unit/test_answerers.py | 2 +- tests/unit/test_engines_init.py | 2 +- tests/unit/test_external_bangs.py | 2 +- tests/unit/test_plugins.py | 2 +- tests/unit/test_preferences.py | 2 +- tests/unit/test_query.py | 2 +- tests/unit/test_results.py | 2 +- tests/unit/test_search.py | 4 +- tests/unit/test_settings_loader.py | 2 +- tests/unit/test_standalone_searx.py | 2 +- tests/unit/test_utils.py | 3 +- tests/unit/test_webadapter.py | 2 +- tests/unit/test_webapp.py | 5 +- tests/unit/test_webutils.py | 2 +- 23 files changed, 161 insertions(+), 157 deletions(-) rename searx/testing.py => tests/robot/__main__.py (50%) rename {searx => tests/robot}/settings_robot.yml (100%) create mode 100644 tests/robot/test_webapp.py diff --git a/manage b/manage index a855512f5..6f6f1a4fe 100755 --- a/manage +++ b/manage @@ -636,7 +636,7 @@ test.coverage() { test.robot() { build_msg TEST 'robot' gecko.driver - PYTHONPATH=. pyenv.cmd python searx/testing.py robot + PYTHONPATH=. pyenv.cmd python -m tests.robot dump_return $? } diff --git a/tests/__init__.py b/tests/__init__.py index cb43fc22a..d4b101cc4 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,47 @@ import os +import aiounittest + os.environ['SEARX_DEBUG'] = '1' os.environ['SEARX_DISABLE_ETC_SETTINGS'] = '1' os.environ.pop('SEARX_SETTINGS_PATH', None) + + +class SearxTestLayer: + """Base layer for non-robot tests.""" + + __name__ = 'SearxTestLayer' + + @classmethod + def setUp(cls): + pass + + @classmethod + def tearDown(cls): + pass + + @classmethod + def testSetUp(cls): + pass + + @classmethod + def testTearDown(cls): + pass + + +class SearxTestCase(aiounittest.AsyncTestCase): + """Base test case for non-robot tests.""" + + layer = SearxTestLayer + + def setattr4test(self, obj, attr, value): + """ + setattr(obj, attr, value) + but reset to the previous value in the cleanup. + """ + previous_value = getattr(obj, attr) + + def cleanup_patch(): + setattr(obj, attr, previous_value) + self.addCleanup(cleanup_patch) + setattr(obj, attr, value) diff --git a/tests/robot/__init__.py b/tests/robot/__init__.py index 7c08a91bd..e69de29bb 100644 --- a/tests/robot/__init__.py +++ b/tests/robot/__init__.py @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- - -from time import sleep - -url = "http://localhost:11111/" - - -def test_index(browser): - # Visit URL - browser.visit(url) - assert browser.is_text_present('about') - - -def test_404(browser): - # Visit URL - browser.visit(url + 'missing_link') - assert browser.is_text_present('Page not found') - - -def test_about(browser): - browser.visit(url) - browser.click_link_by_text('about') - assert browser.is_text_present('Why use it?') - - -def test_preferences(browser): - browser.visit(url) - browser.click_link_by_text('preferences') - assert browser.is_text_present('Preferences') - assert browser.is_text_present('Cookies') - - assert browser.is_element_present_by_xpath('//label[@for="checkbox_dummy"]') - - -def test_preferences_engine_select(browser): - browser.visit(url) - browser.click_link_by_text('preferences') - - assert browser.is_element_present_by_xpath('//a[@href="#tab_engine"]') - browser.find_by_xpath('//a[@href="#tab_engine"]').first.click() - - assert not browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked - browser.find_by_xpath('//label[@for="engine_general_dummy__general"]').first.check() - browser.find_by_xpath('//input[@value="save"]').first.click() - - # waiting for the redirect - without this the test is flaky.. - sleep(1) - - browser.visit(url) - browser.click_link_by_text('preferences') - browser.find_by_xpath('//a[@href="#tab_engine"]').first.click() - - assert browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked - - -def test_preferences_locale(browser): - browser.visit(url) - browser.click_link_by_text('preferences') - - browser.find_by_xpath('//a[@href="#tab_ui"]').first.click() - browser.select('locale', 'hu') - browser.find_by_xpath('//input[@value="save"]').first.click() - - # waiting for the redirect - without this the test is flaky.. - sleep(1) - - browser.visit(url) - browser.click_link_by_text('beállítások') - browser.is_text_present('Beállítások') - - -def test_search(browser): - browser.visit(url) - browser.fill('q', 'test search query') - browser.find_by_xpath('//button[@type="submit"]').first.click() - assert browser.is_text_present('didn\'t find any results') diff --git a/searx/testing.py b/tests/robot/__main__.py similarity index 50% rename from searx/testing.py rename to tests/robot/__main__.py index b31ba8997..e435f512a 100644 --- a/searx/testing.py +++ b/tests/robot/__main__.py @@ -1,40 +1,19 @@ -# -*- coding: utf-8 -*- # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint """Shared testing code.""" # pylint: disable=missing-function-docstring +import sys import os import subprocess import traceback - -from os.path import dirname, join, abspath, realpath +import pathlib from splinter import Browser -import aiounittest - -class SearxTestLayer: - """Base layer for non-robot tests.""" - - __name__ = 'SearxTestLayer' - - @classmethod - def setUp(cls): - pass - - @classmethod - def tearDown(cls): - pass - - @classmethod - def testSetUp(cls): - pass - - @classmethod - def testTearDown(cls): - pass +import tests as searx_tests +from tests.robot import test_webapp class SearxRobotLayer(): @@ -43,8 +22,10 @@ class SearxRobotLayer(): def setUp(self): os.setpgrp() # create new process group, become its leader + tests_path = pathlib.Path(searx_tests.__file__).resolve().parent + # get program paths - webapp = join(abspath(dirname(realpath(__file__))), 'webapp.py') + webapp = str(tests_path.parent / 'searx' / 'webapp.py') exe = 'python' # The Flask app is started by Flask.run(...), don't enable Flask's debug @@ -57,8 +38,7 @@ class SearxRobotLayer(): os.environ['SEARX_DEBUG'] = '0' # set robot settings path - os.environ['SEARX_SETTINGS_PATH'] = abspath( - dirname(__file__) + '/settings_robot.yml') + os.environ['SEARX_SETTINGS_PATH'] = str(tests_path / 'robot' / 'settings_robot.yml') # run the server self.server = subprocess.Popen( # pylint: disable=consider-using-with @@ -75,7 +55,6 @@ class SearxRobotLayer(): del os.environ['SEARX_SETTINGS_PATH'] -# SEARXROBOTLAYER = SearxRobotLayer() def run_robot_tests(tests): print('Running {0} tests'.format(len(tests))) for test in tests: @@ -83,38 +62,17 @@ def run_robot_tests(tests): test(browser) -class SearxTestCase(aiounittest.AsyncTestCase): - """Base test case for non-robot tests.""" - - layer = SearxTestLayer - - def setattr4test(self, obj, attr, value): - """ - setattr(obj, attr, value) - but reset to the previous value in the cleanup. - """ - previous_value = getattr(obj, attr) - - def cleanup_patch(): - setattr(obj, attr, previous_value) - self.addCleanup(cleanup_patch) - setattr(obj, attr, value) +def main(): + test_layer = SearxRobotLayer() + try: + test_layer.setUp() + run_robot_tests([getattr(test_webapp, x) for x in dir(test_webapp) if x.startswith('test_')]) + except Exception: # pylint: disable=broad-except + print('Error occured: {0}'.format(traceback.format_exc())) + sys.exit(1) + finally: + test_layer.tearDown() if __name__ == '__main__': - import sys - # test cases - from tests import robot - - base_dir = abspath(join(dirname(__file__), '../tests')) - if sys.argv[1] == 'robot': - test_layer = SearxRobotLayer() - errors = False - try: - test_layer.setUp() - run_robot_tests([getattr(robot, x) for x in dir(robot) if x.startswith('test_')]) - except Exception: # pylint: disable=broad-except - errors = True - print('Error occured: {0}'.format(traceback.format_exc())) - test_layer.tearDown() - sys.exit(1 if errors else 0) + main() diff --git a/searx/settings_robot.yml b/tests/robot/settings_robot.yml similarity index 100% rename from searx/settings_robot.yml rename to tests/robot/settings_robot.yml diff --git a/tests/robot/test_webapp.py b/tests/robot/test_webapp.py new file mode 100644 index 000000000..cc1550c36 --- /dev/null +++ b/tests/robot/test_webapp.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +# pylint: disable=missing-module-docstring,missing-function-docstring + +from time import sleep + +url = "http://localhost:11111/" + + +def test_index(browser): + # Visit URL + browser.visit(url) + assert browser.is_text_present('about') + + +def test_404(browser): + # Visit URL + browser.visit(url + 'missing_link') + assert browser.is_text_present('Page not found') + + +def test_about(browser): + browser.visit(url) + browser.click_link_by_text('about') + assert browser.is_text_present('Why use it?') + + +def test_preferences(browser): + browser.visit(url) + browser.click_link_by_text('preferences') + assert browser.is_text_present('Preferences') + assert browser.is_text_present('Cookies') + + assert browser.is_element_present_by_xpath('//label[@for="checkbox_dummy"]') + + +def test_preferences_engine_select(browser): + browser.visit(url) + browser.click_link_by_text('preferences') + + assert browser.is_element_present_by_xpath('//a[@href="#tab_engine"]') + browser.find_by_xpath('//a[@href="#tab_engine"]').first.click() + + assert not browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked + browser.find_by_xpath('//label[@for="engine_general_dummy__general"]').first.check() + browser.find_by_xpath('//input[@value="save"]').first.click() + + # waiting for the redirect - without this the test is flaky.. + sleep(1) + + browser.visit(url) + browser.click_link_by_text('preferences') + browser.find_by_xpath('//a[@href="#tab_engine"]').first.click() + + assert browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked + + +def test_preferences_locale(browser): + browser.visit(url) + browser.click_link_by_text('preferences') + + browser.find_by_xpath('//a[@href="#tab_ui"]').first.click() + browser.select('locale', 'hu') + browser.find_by_xpath('//input[@value="save"]').first.click() + + # waiting for the redirect - without this the test is flaky.. + sleep(1) + + browser.visit(url) + browser.click_link_by_text('beállítások') + browser.is_text_present('Beállítások') + + +def test_search(browser): + browser.visit(url) + browser.fill('q', 'test search query') + browser.find_by_xpath('//button[@type="submit"]').first.click() + assert browser.is_text_present('didn\'t find any results') diff --git a/tests/unit/engines/test_command.py b/tests/unit/engines/test_command.py index 8f88b617e..d2bb123f7 100644 --- a/tests/unit/engines/test_command.py +++ b/tests/unit/engines/test_command.py @@ -16,7 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from searx.engines import command as command_engine -from searx.testing import SearxTestCase +from tests import SearxTestCase class TestCommandEngine(SearxTestCase): diff --git a/tests/unit/engines/test_xpath.py b/tests/unit/engines/test_xpath.py index 963a44a25..287beeab4 100644 --- a/tests/unit/engines/test_xpath.py +++ b/tests/unit/engines/test_xpath.py @@ -2,7 +2,7 @@ from collections import defaultdict import mock from searx.engines import xpath -from searx.testing import SearxTestCase +from tests import SearxTestCase class TestXpathEngine(SearxTestCase): diff --git a/tests/unit/network/test_network.py b/tests/unit/network/test_network.py index e8d33240a..0805edc19 100644 --- a/tests/unit/network/test_network.py +++ b/tests/unit/network/test_network.py @@ -5,7 +5,7 @@ from mock import patch import httpx from searx.network.network import Network, NETWORKS, initialize -from searx.testing import SearxTestCase +from tests import SearxTestCase class TestNetwork(SearxTestCase): diff --git a/tests/unit/test_answerers.py b/tests/unit/test_answerers.py index 73d8d26f2..1119b697d 100644 --- a/tests/unit/test_answerers.py +++ b/tests/unit/test_answerers.py @@ -3,7 +3,7 @@ from mock import Mock from searx.answerers import answerers -from searx.testing import SearxTestCase +from tests import SearxTestCase class AnswererTest(SearxTestCase): diff --git a/tests/unit/test_engines_init.py b/tests/unit/test_engines_init.py index d2aee41f2..dffeaf8e8 100644 --- a/tests/unit/test_engines_init.py +++ b/tests/unit/test_engines_init.py @@ -1,5 +1,5 @@ -from searx.testing import SearxTestCase from searx import settings, engines +from tests import SearxTestCase class TestEnginesInit(SearxTestCase): diff --git a/tests/unit/test_external_bangs.py b/tests/unit/test_external_bangs.py index 6a9686782..f320d3037 100644 --- a/tests/unit/test_external_bangs.py +++ b/tests/unit/test_external_bangs.py @@ -1,6 +1,6 @@ from searx.external_bang import get_node, resolve_bang_definition, get_bang_url, get_bang_definition_and_autocomplete from searx.search import SearchQuery, EngineRef -from searx.testing import SearxTestCase +from tests import SearxTestCase TEST_DB = { diff --git a/tests/unit/test_plugins.py b/tests/unit/test_plugins.py index 9ef4cd692..245a7566b 100644 --- a/tests/unit/test_plugins.py +++ b/tests/unit/test_plugins.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from searx.testing import SearxTestCase from searx import plugins from mock import Mock +from tests import SearxTestCase def get_search_mock(query, **kwargs): diff --git a/tests/unit/test_preferences.py b/tests/unit/test_preferences.py index 510d4985a..903b9b54d 100644 --- a/tests/unit/test_preferences.py +++ b/tests/unit/test_preferences.py @@ -1,6 +1,6 @@ from searx.preferences import (EnumStringSetting, MapSetting, MissingArgumentException, SearchLanguageSetting, MultipleChoiceSetting, PluginsSetting, ValidationException) -from searx.testing import SearxTestCase +from tests import SearxTestCase class PluginStub: diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index dd7ff0766..edb0a18f7 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -1,7 +1,7 @@ from searx import settings from searx.engines import load_engines from searx.query import RawTextQuery -from searx.testing import SearxTestCase +from tests import SearxTestCase TEST_ENGINES = [ diff --git a/tests/unit/test_results.py b/tests/unit/test_results.py index 274b5b37a..07d170130 100644 --- a/tests/unit/test_results.py +++ b/tests/unit/test_results.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from searx.results import ResultContainer -from searx.testing import SearxTestCase +from tests import SearxTestCase def fake_result(url='https://aa.bb/cc?dd=ee#ff', diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index b0988a0c8..c7f15a681 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -from searx.testing import SearxTestCase +import searx.search from searx.search import SearchQuery, EngineRef from searx import settings -import searx.search +from tests import SearxTestCase SAFESEARCH = 0 diff --git a/tests/unit/test_settings_loader.py b/tests/unit/test_settings_loader.py index a3257ee4b..ad9fc6e95 100644 --- a/tests/unit/test_settings_loader.py +++ b/tests/unit/test_settings_loader.py @@ -3,9 +3,9 @@ from os.path import dirname, join, abspath from unittest.mock import patch -from searx.testing import SearxTestCase from searx.exceptions import SearxSettingsException from searx import settings_loader +from tests import SearxTestCase test_dir = abspath(dirname(__file__)) diff --git a/tests/unit/test_standalone_searx.py b/tests/unit/test_standalone_searx.py index a69353c03..a5f17c08f 100644 --- a/tests/unit/test_standalone_searx.py +++ b/tests/unit/test_standalone_searx.py @@ -8,8 +8,8 @@ from mock import Mock, patch from nose2.tools import params from searx.search import SearchQuery, EngineRef, initialize -from searx.testing import SearxTestCase from searx_extra import standalone_searx as sas +from tests import SearxTestCase class StandaloneSearx(SearxTestCase): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 91ec2499d..e9c247382 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2,10 +2,11 @@ import lxml.etree from lxml import html -from searx.testing import SearxTestCase from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException from searx import utils +from tests import SearxTestCase + class TestUtils(SearxTestCase): diff --git a/tests/unit/test_webadapter.py b/tests/unit/test_webadapter.py index 32417f019..9d8ff5f28 100644 --- a/tests/unit/test_webadapter.py +++ b/tests/unit/test_webadapter.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from searx.testing import SearxTestCase from searx.preferences import Preferences from searx.engines import engines import searx.search from searx.search import EngineRef from searx.webadapter import validate_engineref_list +from tests import SearxTestCase PRIVATE_ENGINE_NAME = 'general private offline' diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index 8598113f7..c5ec1ca97 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -3,9 +3,10 @@ import json from urllib.parse import ParseResult from mock import Mock -from searx.testing import SearxTestCase -from searx.search import Search + import searx.search.processors +from searx.search import Search +from tests import SearxTestCase class ViewsTestCase(SearxTestCase): diff --git a/tests/unit/test_webutils.py b/tests/unit/test_webutils.py index 023374b04..6da39a071 100644 --- a/tests/unit/test_webutils.py +++ b/tests/unit/test_webutils.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import mock -from searx.testing import SearxTestCase from searx import webutils +from tests import SearxTestCase class TestWebUtils(SearxTestCase): From 3c377c0aa84ae6c94660ff19da9a8a676aa39f09 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 2 Sep 2021 16:15:22 +0200 Subject: [PATCH 2/4] [mod] searx.webapp: code clean up --- searx/webapp.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/searx/webapp.py b/searx/webapp.py index b935424f1..e29b7f442 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -49,13 +49,14 @@ from flask_babel import ( format_decimal, ) -from searx import logger -from searx import get_setting from searx import ( + logger, + get_setting, settings, searx_debug, ) from searx.settings_defaults import OUTPUT_FORMATS +from searx.settings_loader import get_default_settings_path from searx.exceptions import SearxParameterException from searx.engines import ( categories, @@ -91,8 +92,10 @@ from searx.preferences import ( ValidationException, LANGUAGE_CODES, ) -from searx.answerers import answerers -from searx.answerers import ask +from searx.answerers import ( + answerers, + ask, +) from searx.metrics import ( get_engines_stats, get_engine_errors, @@ -110,7 +113,6 @@ from searx.locales import LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES from searx.search import SearchWithPlugins, initialize as search_initialize from searx.network import stream as http_stream, set_context_network_name from searx.search.checker import get_result as checker_get_result -from searx.settings_loader import get_default_settings_path logger = logger.getChild('webapp') From a7d781c49d17c290387a7de983d5fba726e60844 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 2 Sep 2021 16:19:18 +0200 Subject: [PATCH 3/4] [mod] remove remaining references to locales in settings.yml see #247 --- searx/settings_defaults.py | 1 - tests/robot/settings_robot.yml | 4 ---- 2 files changed, 5 deletions(-) diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index e3955d97b..42d88f2bc 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -197,7 +197,6 @@ SCHEMA = { 'off_when_debug': SettingsValue(bool, True), }, 'engines': SettingsValue(list, []), - 'locales': SettingsValue(dict, {'en': 'English'}), 'doi_resolvers': { }, } diff --git a/tests/robot/settings_robot.yml b/tests/robot/settings_robot.yml index ca65449f7..e0d01bb37 100644 --- a/tests/robot/settings_robot.yml +++ b/tests/robot/settings_robot.yml @@ -44,10 +44,6 @@ engines: categories: dummy shortcut: dd -locales: - en: English - hu: Magyar - doi_resolvers: oadoi.org: 'https://oadoi.org/' doi.org: 'https://doi.org/' From a14ea6396e8e55b056065e6f0ce2032067e11385 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Thu, 2 Sep 2021 16:24:20 +0200 Subject: [PATCH 4/4] [mod] remove searx.raise_for_httperror Not used anymore --- searx/raise_for_httperror/__init__.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 searx/raise_for_httperror/__init__.py diff --git a/searx/raise_for_httperror/__init__.py b/searx/raise_for_httperror/__init__.py deleted file mode 100644 index b133da507..000000000 --- a/searx/raise_for_httperror/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# compatibility with searx/searx -from searx.network import raise_for_httperror