mirror of
https://github.com/searxng/searxng.git
synced 2024-08-21 13:50:16 +02:00
commit
aaae9a209e
16
.github/workflows/data-update.yml
vendored
16
.github/workflows/data-update.yml
vendored
@ -11,12 +11,12 @@ jobs:
|
|||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
fetch:
|
fetch:
|
||||||
- ahmia_blacklist
|
- update_ahmia_blacklist.py
|
||||||
- currencies
|
- update_currencies.py
|
||||||
- external_bangs
|
- update_external_bangs.py
|
||||||
- firefox_version
|
- update_firefox_version.py
|
||||||
- languages
|
- update_languages.py
|
||||||
- wikidata_units
|
- update_wikidata_units.py
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
@ -45,10 +45,10 @@ jobs:
|
|||||||
|
|
||||||
- name: Fetch data
|
- name: Fetch data
|
||||||
env:
|
env:
|
||||||
FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py
|
FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
|
||||||
run: |
|
run: |
|
||||||
source local/py3/bin/activate
|
source local/py3/bin/activate
|
||||||
python $FETCH_SCRIPT
|
$FETCH_SCRIPT
|
||||||
|
|
||||||
- name: Create Pull Request
|
- name: Create Pull Request
|
||||||
id: cpr
|
id: cpr
|
||||||
|
4
Makefile
4
Makefile
@ -195,8 +195,8 @@ PYLINT_FILES=\
|
|||||||
searx/engines/google_videos.py \
|
searx/engines/google_videos.py \
|
||||||
searx/engines/google_images.py \
|
searx/engines/google_images.py \
|
||||||
searx/engines/mediathekviewweb.py \
|
searx/engines/mediathekviewweb.py \
|
||||||
utils/fetch_external_bangs.py \
|
searx/engines/google_scholar.py \
|
||||||
searx/engines/google_scholar.py
|
searx_extra/update/update_external_bangs.py
|
||||||
|
|
||||||
test.pylint: pyenvinstall
|
test.pylint: pyenvinstall
|
||||||
$(call cmd,pylint,$(PYLINT_FILES))
|
$(call cmd,pylint,$(PYLINT_FILES))
|
||||||
|
@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.
|
|||||||
user/index
|
user/index
|
||||||
admin/index
|
admin/index
|
||||||
dev/index
|
dev/index
|
||||||
|
searx_extra/index
|
||||||
utils/index
|
utils/index
|
||||||
blog/index
|
blog/index
|
||||||
|
|
||||||
|
14
docs/searx_extra/index.rst
Normal file
14
docs/searx_extra/index.rst
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
.. _searx_extra:
|
||||||
|
|
||||||
|
======================================================
|
||||||
|
Tooling box ``searx_extra`` for developers and users
|
||||||
|
======================================================
|
||||||
|
|
||||||
|
In the folder :origin:`searx_extra/` we maintain some tools useful for
|
||||||
|
developers and users.
|
||||||
|
|
||||||
|
.. toctree::
|
||||||
|
:maxdepth: 2
|
||||||
|
:caption: Contents
|
||||||
|
|
||||||
|
standalone_searx.py
|
9
docs/searx_extra/standalone_searx.py.rst
Normal file
9
docs/searx_extra/standalone_searx.py.rst
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
|
||||||
|
.. _standalone_searx.py:
|
||||||
|
|
||||||
|
===================================
|
||||||
|
``searx_extra/standalone_searx.py``
|
||||||
|
===================================
|
||||||
|
|
||||||
|
.. automodule:: searx_extra.standalone_searx
|
||||||
|
:members:
|
@ -1,12 +1,11 @@
|
|||||||
.. _searx_utils:
|
.. _searx_utils:
|
||||||
.. _toolboxing:
|
.. _toolboxing:
|
||||||
|
|
||||||
=======================
|
========================================
|
||||||
Tooling box ``utils/*``
|
Tooling box ``utils`` for administrators
|
||||||
=======================
|
========================================
|
||||||
|
|
||||||
In the folder :origin:`utils/` we maintain some tools useful for admins and
|
In the folder :origin:`utils/` we maintain some tools useful for administrators.
|
||||||
developers.
|
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 2
|
:maxdepth: 2
|
||||||
@ -16,7 +15,6 @@ developers.
|
|||||||
filtron.sh
|
filtron.sh
|
||||||
morty.sh
|
morty.sh
|
||||||
lxc.sh
|
lxc.sh
|
||||||
standalone_searx.py
|
|
||||||
|
|
||||||
.. _toolboxing common:
|
.. _toolboxing common:
|
||||||
|
|
||||||
|
@ -1,11 +0,0 @@
|
|||||||
|
|
||||||
.. _standalone_searx.py:
|
|
||||||
|
|
||||||
=============================
|
|
||||||
``utils/standalone_searx.py``
|
|
||||||
=============================
|
|
||||||
|
|
||||||
.. automodule:: standalone_searx
|
|
||||||
:members:
|
|
||||||
|
|
||||||
|
|
0
searx_extra/__init__.py
Normal file
0
searx_extra/__init__.py
Normal file
@ -15,7 +15,7 @@ Example to use this script:
|
|||||||
|
|
||||||
.. code:: bash
|
.. code:: bash
|
||||||
|
|
||||||
$ python3 utils/standalone_searx.py rain
|
$ python3 searx_extra/standalone_searx.py rain
|
||||||
|
|
||||||
Example to run it from python:
|
Example to run it from python:
|
||||||
|
|
0
searx_extra/update/__init__.py
Normal file
0
searx_extra/update/__init__.py
Normal file
@ -4,11 +4,8 @@
|
|||||||
# More info in https://ahmia.fi/blacklist/
|
# More info in https://ahmia.fi/blacklist/
|
||||||
|
|
||||||
# set path
|
# set path
|
||||||
from sys import path
|
from os.path import join
|
||||||
from os.path import realpath, dirname, join
|
|
||||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
|
||||||
|
|
||||||
#
|
|
||||||
import requests
|
import requests
|
||||||
from searx import searx_dir
|
from searx import searx_dir
|
||||||
|
|
1
utils/fetch_currencies.py → searx_extra/update/update_currencies.py
Normal file → Executable file
1
utils/fetch_currencies.py → searx_extra/update/update_currencies.py
Normal file → Executable file
@ -7,7 +7,6 @@ import json
|
|||||||
# set path
|
# set path
|
||||||
from sys import path
|
from sys import path
|
||||||
from os.path import realpath, dirname, join
|
from os.path import realpath, dirname, join
|
||||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
|
||||||
|
|
||||||
from searx import searx_dir, settings
|
from searx import searx_dir, settings
|
||||||
from searx.engines.wikidata import send_wikidata_query
|
from searx.engines.wikidata import send_wikidata_query
|
6
utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.py
Normal file → Executable file
6
utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.py
Normal file → Executable file
@ -3,13 +3,9 @@
|
|||||||
import sys
|
import sys
|
||||||
import json
|
import json
|
||||||
from urllib.parse import quote, urlparse
|
from urllib.parse import quote, urlparse
|
||||||
from os.path import realpath, dirname
|
import detect_language
|
||||||
import cld3
|
|
||||||
from lxml.html import fromstring
|
from lxml.html import fromstring
|
||||||
|
|
||||||
# set path
|
|
||||||
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
|
||||||
|
|
||||||
from searx.engines.wikidata import send_wikidata_query
|
from searx.engines.wikidata import send_wikidata_query
|
||||||
from searx.utils import extract_text
|
from searx.utils import extract_text
|
||||||
import searx
|
import searx
|
@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION
|
|||||||
"""
|
"""
|
||||||
# pylint: disable=C0116
|
# pylint: disable=C0116
|
||||||
|
|
||||||
import sys
|
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from os.path import realpath, dirname, join
|
from os.path import join
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
# set path
|
|
||||||
sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
|
||||||
|
|
||||||
from searx import searx_dir # pylint: disable=E0401 C0413
|
from searx import searx_dir # pylint: disable=E0401 C0413
|
||||||
|
|
||||||
|
|
@ -1,14 +1,9 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
# set path
|
|
||||||
from sys import path
|
|
||||||
from os.path import realpath, dirname, join
|
|
||||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
|
||||||
|
|
||||||
#
|
|
||||||
import json
|
import json
|
||||||
import requests
|
import requests
|
||||||
import re
|
import re
|
||||||
|
from os.path import dirname, join
|
||||||
from urllib.parse import urlparse, urljoin
|
from urllib.parse import urlparse, urljoin
|
||||||
from distutils.version import LooseVersion, StrictVersion
|
from distutils.version import LooseVersion, StrictVersion
|
||||||
from lxml import html
|
from lxml import html
|
4
utils/fetch_languages.py → searx_extra/update/update_languages.py
Normal file → Executable file
4
utils/fetch_languages.py → searx_extra/update/update_languages.py
Normal file → Executable file
@ -1,4 +1,4 @@
|
|||||||
# -*- coding: utf-8 -*-
|
#!/usr/bin/env python
|
||||||
|
|
||||||
# This script generates languages.py from intersecting each engine's supported languages.
|
# This script generates languages.py from intersecting each engine's supported languages.
|
||||||
#
|
#
|
||||||
@ -7,11 +7,9 @@
|
|||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from pprint import pformat
|
from pprint import pformat
|
||||||
from sys import path
|
|
||||||
from babel import Locale, UnknownLocaleError
|
from babel import Locale, UnknownLocaleError
|
||||||
from babel.languages import get_global
|
from babel.languages import get_global
|
||||||
|
|
||||||
path.append('../searx') # noqa
|
|
||||||
from searx import settings, searx_dir
|
from searx import settings, searx_dir
|
||||||
from searx.engines import initialize_engines, engines
|
from searx.engines import initialize_engines, engines
|
||||||
|
|
4
utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.py
Normal file → Executable file
4
utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.py
Normal file → Executable file
@ -4,9 +4,7 @@ import json
|
|||||||
import collections
|
import collections
|
||||||
|
|
||||||
# set path
|
# set path
|
||||||
from sys import path
|
from os.path import join
|
||||||
from os.path import realpath, dirname, join
|
|
||||||
path.append(realpath(dirname(realpath(__file__)) + '/../'))
|
|
||||||
|
|
||||||
from searx import searx_dir
|
from searx import searx_dir
|
||||||
from searx.engines.wikidata import send_wikidata_query
|
from searx.engines.wikidata import send_wikidata_query
|
2
setup.py
2
setup.py
@ -41,7 +41,7 @@ setup(
|
|||||||
author='Adam Tauber',
|
author='Adam Tauber',
|
||||||
author_email='asciimoo@gmail.com',
|
author_email='asciimoo@gmail.com',
|
||||||
license='GNU Affero General Public License',
|
license='GNU Affero General Public License',
|
||||||
packages=find_packages(exclude=["tests*"]),
|
packages=find_packages(exclude=["tests*", "searx_extra"]),
|
||||||
zip_safe=False,
|
zip_safe=False,
|
||||||
install_requires=requirements,
|
install_requires=requirements,
|
||||||
extras_require={
|
extras_require={
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""Test utils/standalone_searx.py"""
|
"""Test utils/standalone_searx.py"""
|
||||||
import datetime
|
import datetime
|
||||||
import importlib.util
|
|
||||||
import io
|
import io
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
@ -10,16 +9,7 @@ from nose2.tools import params
|
|||||||
|
|
||||||
from searx.search import SearchQuery, EngineRef, initialize
|
from searx.search import SearchQuery, EngineRef, initialize
|
||||||
from searx.testing import SearxTestCase
|
from searx.testing import SearxTestCase
|
||||||
|
from searx_extra import standalone_searx as sas
|
||||||
|
|
||||||
def get_standalone_searx_module():
|
|
||||||
"""Get standalone_searx module."""
|
|
||||||
module_name = 'utils.standalone_searx'
|
|
||||||
filename = 'utils/standalone_searx.py'
|
|
||||||
spec = importlib.util.spec_from_file_location(module_name, filename)
|
|
||||||
sas = importlib.util.module_from_spec(spec)
|
|
||||||
spec.loader.exec_module(sas)
|
|
||||||
return sas
|
|
||||||
|
|
||||||
|
|
||||||
class StandaloneSearx(SearxTestCase):
|
class StandaloneSearx(SearxTestCase):
|
||||||
@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):
|
|||||||
|
|
||||||
def test_parse_argument_no_args(self):
|
def test_parse_argument_no_args(self):
|
||||||
"""Test parse argument without args."""
|
"""Test parse argument without args."""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
with patch.object(sys, 'argv', ['standalone_searx']), \
|
with patch.object(sys, 'argv', ['standalone_searx']), \
|
||||||
self.assertRaises(SystemExit):
|
self.assertRaises(SystemExit):
|
||||||
sys.stderr = io.StringIO()
|
sys.stderr = io.StringIO()
|
||||||
@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):
|
|||||||
|
|
||||||
def test_parse_argument_basic_args(self):
|
def test_parse_argument_basic_args(self):
|
||||||
"""Test parse argument with basic args."""
|
"""Test parse argument with basic args."""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
query = 'red box'
|
query = 'red box'
|
||||||
exp_dict = {
|
exp_dict = {
|
||||||
'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
|
'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,
|
||||||
@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):
|
|||||||
|
|
||||||
def test_to_dict(self):
|
def test_to_dict(self):
|
||||||
"""test to_dict."""
|
"""test to_dict."""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
sas.to_dict(
|
sas.to_dict(
|
||||||
sas.get_search_query(sas.parse_argument(['red box']))),
|
sas.get_search_query(sas.parse_argument(['red box']))),
|
||||||
@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):
|
|||||||
|
|
||||||
def test_to_dict_with_mock(self):
|
def test_to_dict_with_mock(self):
|
||||||
"""test to dict."""
|
"""test to dict."""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
with patch.object(sas.searx.search, 'Search') as mock_s:
|
with patch.object(sas.searx.search, 'Search') as mock_s:
|
||||||
m_search = mock_s().search()
|
m_search = mock_s().search()
|
||||||
m_sq = Mock()
|
m_sq = Mock()
|
||||||
@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):
|
|||||||
|
|
||||||
def test_get_search_query(self):
|
def test_get_search_query(self):
|
||||||
"""test get_search_query."""
|
"""test get_search_query."""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
args = sas.parse_argument(['rain', ])
|
args = sas.parse_argument(['rain', ])
|
||||||
search_q = sas.get_search_query(args)
|
search_q = sas.get_search_query(args)
|
||||||
self.assertTrue(search_q)
|
self.assertTrue(search_q)
|
||||||
@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):
|
|||||||
|
|
||||||
def test_no_parsed_url(self):
|
def test_no_parsed_url(self):
|
||||||
"""test no_parsed_url func"""
|
"""test no_parsed_url func"""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
|
sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),
|
||||||
[{}]
|
[{}]
|
||||||
@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):
|
|||||||
)
|
)
|
||||||
def test_json_serial(self, arg, exp_res):
|
def test_json_serial(self, arg, exp_res):
|
||||||
"""test json_serial func"""
|
"""test json_serial func"""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
self.assertEqual(sas.json_serial(arg), exp_res)
|
self.assertEqual(sas.json_serial(arg), exp_res)
|
||||||
|
|
||||||
def test_json_serial_error(self):
|
def test_json_serial_error(self):
|
||||||
"""test error on json_serial."""
|
"""test error on json_serial."""
|
||||||
sas = get_standalone_searx_module()
|
|
||||||
with self.assertRaises(TypeError):
|
with self.assertRaises(TypeError):
|
||||||
sas.json_serial('a')
|
sas.json_serial('a')
|
||||||
|
Loading…
Reference in New Issue
Block a user