kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-22 12:10:11 +01:00
Code

Merge branch 'master' into bug/oscar-theme

Browse Source
This commit is contained in:
Markus Heiser 2020-01-06 08:05:29 +00:00 committed by GitHub
commit 1d86d0054f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 2253 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -8,6 +8,9 @@ PYOBJECTS = searx
DOC = docs
PY_SETUP_EXTRAS ?= \[test\]
PYDIST=./dist/py
PYBUILD=./build/py
include utils/makefile.include
include utils/makefile.python
include utils/makefile.sphinx
@ -23,6 +26,7 @@ help:
@echo ' install - developer install (./local)'
@echo ' uninstall - uninstall (./local)'
@echo ' gh-pages - build docs & deploy on gh-pages branch'
@echo ' clean - drop builds and environments'
@echo ''
@$(MAKE) -s -f utils/makefile.include make-help
@echo ''

6
README.rst
View File

@ -23,13 +23,13 @@ Go to the `searx-docker <https://github.com/searx/searx-docker>`__ project.
Without Docker
------
For all the details, follow this `step by step installation <https://asciimoo.github.io/searx/dev/install/installation.html>`__.
For all of the details, follow this `step by step installation <https://asciimoo.github.io/searx/dev/install/installation.html>`__.
Note: the documentation needs to be updated.
If you are in hurry
If you are in a hurry
------
- clone source:
- clone the source:
``git clone https://github.com/asciimoo/searx.git && cd searx``
- install dependencies: ``./manage.sh update_packages``
- edit your

102
docs/_themes/searx/static/searx.css vendored
View File

@ -24,7 +24,107 @@ p.sidebar-title, .sidebar p {
margin: 6pt;
}
.sidebar li {
.sidebar li,
.hlist li {
list-style-type: disclosure-closed;
}
/* admonitions
*/
div.admonition, div.topic {
background-color: #fafafa;
margin: 8px 0px;
padding: 1em;
border-radius: 3pt 0 0 3pt;
border-top: none;
border-right: none;
border-bottom: none;
border-left: 5pt solid #ccc;
}
p.admonition-title:after {
content: none;
}
.admonition.hint { border-color: #416dc0b0; }
.admonition.note { border-color: #6c856cb0; }
.admonition.tip { border-color: #85c5c2b0; }
.admonition.attention { border-color: #ecec97b0; }
.admonition.caution { border-color: #a6c677b0; }
.admonition.danger { border-color: #d46262b0; }
.admonition.important { border-color: #dfa3a3b0; }
.admonition.error { border-color: red; }
.admonition.warning { border-color: darkred; }
.admonition.admonition-generic-admonition-title {
border-color: #416dc0b0;
}
/* admonitions with (rendered) reST markup examples (:class: rst-example)
*
* .. admonition:: title of the example
* :class: rst-example
* ....
*/
div.rst-example {
background-color: inherit;
margin: 0;
border-top: none;
border-right: 1px solid #ccc;
border-bottom: none;
border-left: none;
border-radius: none;
padding: 0;
}
div.rst-example > p.admonition-title {
font-family: Sans Serif;
font-style: italic;
font-size: 0.8em;
display: block;
border-bottom: 1px solid #ccc;
padding: 0.5em 1em;
text-align: right;
}
/* code block in figures
*/
div.highlight pre {
text-align: left;
}
/* Table theme
*/
thead, tfoot {
background-color: #fff;
}
th:hover, td:hover {
background-color: #ffc;
}
thead th, tfoot th, tfoot td, tbody th {
background-color: #fffaef;
}
tbody tr:nth-child(odd) {
background-color: #fff;
}
tbody tr:nth-child(even) {
background-color: #fafafa;
}
caption {
font-family: Sans Serif;
padding: 0.5em;
margin: 0.5em 0 0.5em 0;
caption-side: top;
text-align: left;
}

33
docs/admin/arch_public.dot Normal file
View File

@ -0,0 +1,33 @@
digraph G {
node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"];
edge [fontname="Sans"];
browser [label="Browser", shape=Mdiamond];
rp [label="Reverse Proxy", href="url to configure reverse proxy"];
filtron [label="Filtron", href="https://github.com/asciimoo/filtron"];
morty [label="Morty", href="https://github.com/asciimoo/morty"];
static [label="Static files", href="url to configure static files"];
uwsgi [label="uwsgi", href="url to configure uwsgi"]
searx1 [label="Searx #1"];
searx2 [label="Searx #2"];
searx3 [label="Searx #3"];
searx4 [label="Searx #4"];
browser -> rp [label="HTTPS"]
subgraph cluster_searx {
label = "Searx instance" fontname="Sans";
bgcolor="#fafafa";
{ rank=same; static rp };
rp -> morty [label="optional: images and HTML pages proxy"];
rp -> static [label="optional: reverse proxy serves directly static files"];
rp -> filtron [label="HTTP"];
filtron -> uwsgi [label="HTTP"];
uwsgi -> searx1;
uwsgi -> searx2;
uwsgi -> searx3;
uwsgi -> searx4;
}
}

24
docs/admin/architecture.rst Normal file
View File

@ -0,0 +1,24 @@
.. _architecture:
============
Architecture
============
.. sidebar:: Needs work!
This article needs some work / Searx is a collaborative effort. If you have
any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see
:ref:`how to contribute`.
Herein you will find some hints and suggestions about typical architectures of
searx infrastructures.
We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`.
It shows a *reference* setup for public searx instances.
.. _arch public:
.. kernel-figure:: arch_public.dot
:alt: arch_public.dot
Reference architecture of a public searx setup.

103
docs/admin/buildhosts.rst Normal file
View File

@ -0,0 +1,103 @@
.. _buildhosts:
==========
Buildhosts
==========
.. sidebar:: This article needs some work
If you have any contribution send us your :pull:`PR <../pulls>`, see
:ref:`how to contribute`.
To get best results from build, its recommend to install additional packages
on build hosts.
.. _docs build:
Build docs
==========
.. _Graphviz: https://graphviz.gitlab.io
.. _ImageMagick: https://www.imagemagick.org
.. _XeTeX: https://tug.org/xetex/
.. _dvisvgm: https://dvisvgm.de/
.. sidebar:: Sphinx build needs
- ImageMagick_
- Graphviz_
- XeTeX_
- dvisvgm_
Most of the sphinx requirements are installed from :origin:`setup.py` and the
docs can be build from scratch with ``make docs``. For better math and image
processing additional packages are needed. The XeTeX_ needed not only for PDF
creation, its also needed for :ref:`math` when HTML output is build.
To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
as images (``sphinx.ext.imgmath`` extension). If your docs build (``make
docs``) shows warnings like this::
WARNING: dot(1) not found, for better output quality install \
graphviz from http://www.graphviz.org
..
WARNING: LaTeX command 'latex' cannot be run (needed for math \
display), check the imgmath_latex setting
you need to install additional packages on your build host, to get better HTML
output.
.. _system requirements:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code-block:: sh
$ sudo apt install graphviz imagemagick texlive-xetex librsvg2-bin
.. group-tab:: Arch Linux
.. code-block:: sh
$ sudo pacman -S graphviz imagemagick texlive-bin extra/librsvg
.. group-tab:: Fedora / RHEL
.. code-block:: sh
$ sudo dnf install graphviz graphviz-gd texlive-xetex-bin librsvg2-tools
For PDF output you also need:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
$ sudo apt texlive-latex-recommended texlive-extra-utils ttf-dejavu
.. group-tab:: Arch Linux
.. code:: sh
$ sudo pacman -S texlive-core texlive-latexextra ttf-dejavu
.. group-tab:: Fedora / RHEL
.. code:: sh
$ sudo dnf install \
texlive-collection-fontsrecommended texlive-collection-latex \
dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts
.. _system requirements END:
.. literalinclude:: ../conf.py
:language: python
:start-after: # sphinx.ext.imgmath setup
:end-before: # sphinx.ext.imgmath setup END

68
docs/admin/engines.rst Normal file
View File

@ -0,0 +1,68 @@
.. _engines generic:
=======
engines
=======
.. sidebar:: Further reading ..
- :ref:`engines generic`
- :ref:`configured engines`
- :ref:`engine settings`
- :ref:`engine file`
============= =========== ==================== ============
:ref:`engine settings` :ref:`engine file`
------------------------- ---------------------------------
Name (cfg) Categories
------------------------- ---------------------------------
Engine .. Paging support **P**
------------------------- -------------------- ------------
Shortcut **S** Language support **L**
Timeout **TO** Time range support **TR**
Disabled **D** Offline **O**
------------- ----------- -------------------- ------------
Suspend end **SE**
------------- ----------- ---------------------------------
Safe search **SS**
============= =========== =================================
Configuration defaults (at built time):
.. _configured engines:
.. jinja:: webapp
.. flat-table:: Engines configured at built time (defaults)
:header-rows: 1
:stub-columns: 2
* - Name (cfg)
- S
- Engine
- TO
- Categories
- P
- L
- SS
- D
- TR
- O
- SE
{% for name, mod in engines.items() %}
* - {{name}}
- !{{mod.shortcut}}
- {{mod.__name__}}
- {{mod.timeout}}
- {{", ".join(mod.categories)}}
- {{(mod.paging and "y") or ""}}
- {{(mod.language_support and "y") or ""}}
- {{(mod.safesearch and "y") or ""}}
- {{(mod.disabled and "y") or ""}}
- {{(mod.time_range_support and "y") or ""}}
- {{(mod.offline and "y") or ""}}
- {{mod.suspend_end_time}}
{% endfor %}

4
docs/admin/index.rst
View File

@ -7,5 +7,9 @@ Administrator documentation
installation
api
architecture
filtron
morty
engines
plugins
buildhosts

22
docs/admin/installation.rst
View File

@ -114,6 +114,9 @@ content:
# Module to import
module = searx.webapp
# Support running the module from a webserver subdirectory.
route-run = fixpathinfo:
# Virtualenv and python path
virtualenv = /usr/local/searx/searx-ve/
pythonpath = /usr/local/searx/
@ -151,7 +154,10 @@ content:
server {
listen 80;
server_name searx.example.com;
root /usr/local/searx;
root /usr/local/searx/searx;
location /static {
}
location / {
include uwsgi_params;
@ -180,14 +186,13 @@ Add this configuration in the server config file
.. code:: nginx
location = /searx { rewrite ^ /searx/; }
location /searx {
try_files $uri @searx;
location /searx/static {
alias /usr/local/searx/searx/static;
}
location @searx {
location /searx {
uwsgi_param SCRIPT_NAME /searx;
include uwsgi_params;
uwsgi_modifier1 30;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
@ -197,6 +202,10 @@ in case of single-user or low-traffic instances.)
.. code:: nginx
location /searx/static {
alias /usr/local/searx/searx/static;
}
location /searx {
proxy_pass http://127.0.0.1:8888;
proxy_set_header Host $host;
@ -338,4 +347,3 @@ References
* How to: `Setup searx in a couple of hours with a free SSL certificate
<https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__

39
docs/admin/plugins.rst Normal file
View File

@ -0,0 +1,39 @@
.. _plugins generic:
===============
Plugins builtin
===============
.. sidebar:: Further reading ..
- :ref:`dev plugin`
Configuration defaults (at built time):
:DO: Default on
.. _configured plugins:
.. jinja:: webapp
.. flat-table:: Plugins configured at built time (defaults)
:header-rows: 1
:stub-columns: 1
:widths: 3 1 9
* - Name
- DO
- Description
JS & CSS dependencies
{% for plgin in plugins %}
* - {{plgin.name}}
- {{(plgin.default_on and "y") or ""}}
- {{plgin.description}}
{% for dep in (plgin.js_dependencies + plgin.css_dependencies) %}
| ``{{dep}}`` {% endfor %}
{% endfor %}

37
docs/conf.py
View File

@ -14,11 +14,18 @@ project = u'searx'
copyright = u'2015-2019, Adam Tauber, Noémi Ványi'
author = u'Adam Tauber'
release, version = VERSION_STRING, VERSION_STRING
highlight_language = 'none'
# General --------------------------------------------------------------
master_doc = "index"
source_suffix = '.rst'
numfig = True
from searx import webapp
jinja_contexts = {
'webapp': dict(**webapp.__dict__)
}
# usage:: lorem :patch:`f373169` ipsum
extlinks = {}
@ -32,21 +39,40 @@ extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://')
extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
extlinks['search'] = (SEARX_URL + '/%s', '#')
extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ')
extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
#extlinks['role'] = (
# 'https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-%s', '')
extlinks['duref'] = (
'http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html#%s', '')
extlinks['durole'] = (
'http://docutils.sourceforge.net/docs/ref/rst/roles.html#%s', '')
extlinks['dudir'] = (
'http://docutils.sourceforge.net/docs/ref/rst/directives.html#%s', '')
extlinks['ctan'] = (
'https://ctan.org/pkg/%s', 'CTAN: ')
extensions = [
'sphinx.ext.imgmath',
'sphinx.ext.extlinks',
'sphinx.ext.viewcode',
"sphinx.ext.autodoc",
"sphinx.ext.intersphinx",
"pallets_sphinx_themes",
"sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
"sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja
'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive.
'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling.
"sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
]
intersphinx_mapping = {
"python": ("https://docs.python.org/3/", None),
# "flask": ("https://flask.palletsprojects.com/", None),
"flask": ("https://flask.palletsprojects.com/", None),
# "werkzeug": ("https://werkzeug.palletsprojects.com/", None),
# "jinja": ("https://jinja.palletsprojects.com/", None),
"jinja": ("https://jinja.palletsprojects.com/", None),
"linuxdoc" : ("https://return42.github.io/linuxdoc/", None),
"sphinx" : ("https://www.sphinx-doc.org/en/master/", None),
}
issues_github_path = "asciimoo/searx"
@ -54,10 +80,15 @@ issues_github_path = "asciimoo/searx"
# HTML -----------------------------------------------------------------
sys.path.append(os.path.abspath('_themes'))
html_theme_path = ['_themes']
html_theme = "searx"
# sphinx.ext.imgmath setup
html_math_renderer = 'imgmath'
imgmath_image_format = 'svg'
imgmath_font_size = 14
# sphinx.ext.imgmath setup END
html_theme_options = {"index_sidebar_logo": True}
html_context = {
"project_links": [

43
docs/dev/contribution_guide.rst
View File

@ -1,3 +1,5 @@
.. _how to contribute:
=================
How to contribute
=================
@ -48,7 +50,20 @@ Code
====
.. _PEP8: https://www.python.org/dev/peps/pep-0008/
.. _Conventional Commits: https://www.conventionalcommits.org/
.. _Git Commit Good Practice: https://wiki.openstack.org/wiki/GitCommitMessages
.. _Structural split of changes:
https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes
.. _gitmoji: https://gitmoji.carloscuesta.me/
.. _Semantic PR: https://github.com/zeke/semantic-pull-requests
.. sidebar:: Create good commits!
- `Structural split of changes`_
- `Conventional Commits`_
- `Git Commit Good Practice`_
- some like to use: gitmoji_
- not yet active: `Semantic PR`_
In order to submit a patch, please follow the steps below:
@ -57,6 +72,9 @@ In order to submit a patch, please follow the steps below:
- PEP8_ standards apply, except the convention of line length
- Maximum line length is 120 characters
- The cardinal rule for creating good commits is to ensure there is only one
*logical change* per commit / read `Structural split of changes`_
- Check if your code breaks existing tests. If so, update the tests or fix your
code.
@ -64,6 +82,16 @@ In order to submit a patch, please follow the steps below:
- Add yourself to the :origin:`AUTHORS.rst` file.
- Choose meaning full commit messages, read `Conventional Commits`_
.. code::
<type>[optional scope]: <description>
[optional body]
[optional footer(s)]
- Create a pull request.
For more help on getting started with searx development, see :ref:`devquickstart`.
@ -79,6 +107,8 @@ Translation currently takes place on :ref:`transifex <translation>`.
Please, do not update translation files in the repo.
.. _contrib docs:
Documentation
=============
@ -91,7 +121,7 @@ Documentation
The documentation is built using Sphinx_. So in order to be able to generate
the required files, you have to install it on your system. Much easier, use
Makefile our targets.
our :ref:`makefile`.
Here is an example which makes a complete rebuild:
@ -101,6 +131,7 @@ Here is an example which makes a complete rebuild:
...
The HTML pages are in dist/docs.
.. _make docs-live:
live build
----------
@ -110,9 +141,10 @@ live build
It is recommended to assert a complete rebuild before deploying (use
``docs-clean``).
Live build is like WYSIWYG, If you want to edit the documentation, its
recommended to use. The Makefile target ``docs-live`` builds the docs, opens URL
in your favorite browser and rebuilds every time a reST file has been changed.
Live build is like WYSIWYG. If you want to edit the documentation, its
recommended to use. The Makefile target ``docs-live`` builds the docs, opens
URL in your favorite browser and rebuilds every time a reST file has been
changed.
.. code:: sh
@ -123,12 +155,13 @@ in your favorite browser and rebuilds every time a reST file has been changed.
... Start watching changes
.. _deploy on github.io:
deploy on github.io
-------------------
To deploy documentation at :docs:`github.io <.>` use Makefile target
``gh-pages``, which will builds the documentation, clones searx into a sub
:ref:`make gh-pages`, which will builds the documentation, clones searx into a sub
folder ``gh-pages``, cleans it, copies the doc build into and runs all the
needed git add, commit and push:

6
docs/dev/csv_table.txt Normal file
View File

@ -0,0 +1,6 @@
stub col row 1, column, "loremLorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
voluptua."
stub col row 1, "At vero eos et accusam et justo duo dolores et ea rebum. Stet clita
kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.", column
stub col row 1, column, column

2
docs/dev/engine_overview.rst
View File

@ -29,6 +29,7 @@ the ones in the engine file.
It does not matter if an option is stored in the engine file or in the
settings. However, the standard way is the following:
.. _engine file:
engine file
-----------
@ -43,6 +44,7 @@ time_range_support boolean support search time range
offline boolean engine runs offline
======================= =========== ===========================================
.. _engine settings:
settings.yml
------------

3
docs/dev/hello.dot Normal file
View File

@ -0,0 +1,3 @@
graph G {
Hello -- World
}

2
docs/dev/index.rst
View File

@ -11,3 +11,5 @@ Developer documentation
search_api
plugins
translation
makefile
reST

221
docs/dev/makefile.rst Normal file
View File

@ -0,0 +1,221 @@
.. _makefile:
================
Makefile Targets
================
.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction
.. sidebar:: build environment
Before looking deeper at the targets, first read about :ref:`makefile setup`
and :ref:`make pyenv`.
With the aim to simplify development cycles, started with :pull:`1756` a
``Makefile`` based boilerplate was added. If you are not familiar with
Makefiles, we recommend to read gnu-make_ introduction.
The usage is simple, just type ``make {target-name}`` to *build* a target.
Calling the ``help`` target gives a first overview::
$ make help
test - run developer tests
docs - build documentation
docs-live - autobuild HTML documentation while editing
run - run developer instance
install - developer install (./local)
uninstall - uninstall (./local)
gh-pages - build docs & deploy on gh-pages branch
clean - drop builds and environments
...
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
.. _makefile setup:
Setup
=====
.. _git stash: https://git-scm.com/docs/git-stash
The main setup is done in the :origin:`Makefile`::
export GIT_URL=https://github.com/asciimoo/searx
export SEARX_URL=https://searx.me
export DOCS_URL=https://asciimoo.github.io/searx
.. sidebar:: fork & upstream
Commit changes in your (local) branch, fork or whatever, but do not push them
upstream / `git stash`_ is your friend.
:GIT_URL: Changes this, to point to your searx fork.
:SEARX_URL: Changes this, to point to your searx instance.
:DOCS_URL: If you host your own (branded) documentation, change this URL.
.. _make pyenv:
Python environment
==================
.. sidebar:: activate environment
``source ./local/py3/bin/activate``
With Makefile we do no longer need to build up the virualenv manually (as
described in the :ref:`devquickstart` guide). Jump into your git working tree
and release a ``make pyenv``:
.. code:: sh
$ cd ~/searx-clone
$ make pyenv
PYENV usage: source ./local/py3/bin/activate
...
With target ``pyenv`` a development environment (aka virtualenv) was build up in
``./local/py3/``. To make a *developer install* of searx (:origin:`setup.py`)
into this environment, use make target ``install``:
.. code:: sh
$ make install
PYENV usage: source ./local/py3/bin/activate
PYENV using virtualenv from ./local/py3
PYENV install .
You have never to think about intermediate targets like ``pyenv`` or
``install``, the ``Makefile`` chains them as requisites. Just run your main
target.
.. sidebar:: drop environment
To get rid of the existing environment before re-build use :ref:`clean target
<make clean>` first.
If you think, something goes wrong with your ./local environment or you change
the :origin:`setup.py` file (or the requirements listed in
:origin:`requirements-dev.txt` and :origin:`requirements.txt`), you have to call
:ref:`make clean`.
.. _make run:
``make run``
============
To get up a running a developer instance simply call ``make run``. This enables
*debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``
instance, disables *debug* option again and opens the URL in your favorite WEB
browser (:man:`xdg-open`):
.. code:: sh
$ make run
PYENV usage: source ./local/py3/bin/activate
PYENV install .
./local/py3/bin/python ./searx/webapp.py
...
INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)
...
.. _make clean:
``make clean``
==============
Drop all intermediate files, all builds, but keep sources untouched. Includes
target ``pyclean`` which drops ./local environment. Before calling ``make
clean`` stop all processes using :ref:`make pyenv`.
.. code:: sh
$ make clean
CLEAN pyclean
CLEAN clean
.. _make docs:
``make docs docs-live docs-clean``
==================================
We describe the usage of the ``doc*`` targets in the :ref:`How to contribute /
Documentation <contrib docs>` section. If you want to edit the documentation
read our :ref:`make docs-live` section. If you are working in your own brand,
adjust your :ref:`Makefile setup <makefile setup>`.
.. _make gh-pages:
``make gh-pages``
=================
To deploy on github.io first adjust your :ref:`Makefile setup <makefile
setup>`. For any further read :ref:`deploy on github.io`.
.. _make test:
``make test``
=============
Runs a series of tests: ``test.pep8``, ``test.unit``, ``test.robot`` and does
additional :ref:`pylint checks <make pylint>`. You can run tests selective,
e.g.:
.. code:: sh
$ make test.pep8 test.unit
. ./local/py3/bin/activate; ./manage.sh pep8_check
[!] Running pep8 check
. ./local/py3/bin/activate; ./manage.sh unit_tests
[!] Running unit tests
.. _make pylint:
``make pylint``
===============
.. _Pylint: https://www.pylint.org/
Before commiting its recommend to do some (more) linting. Pylint_ is known as
one of the best source-code, bug and quality checker for the Python programming
language. Pylint_ is not yet a quality gate within our searx project (like
:ref:`test.pep8 <make test>` it is), but Pylint_ can help to improve code
quality anyway. The pylint profile we use at searx project is found in
project's root folder :origin:`.pylintrc`.
Code quality is a ongoing process. Don't try to fix all messages from Pylint,
run Pylint and check if your changed lines are bringing up new messages. If so,
fix it. By this, code quality gets incremental better and if there comes the
day, the linting is balanced out, we might decide to add Pylint as a quality
gate.
``make pybuild``
================
.. _PyPi: https://pypi.org/
.. _twine: https://twine.readthedocs.io/en/latest/
Build Python packages in ``./dist/py``.
.. code:: sh
$ make pybuild
...
BUILD pybuild
running sdist
running egg_info
...
$ ls ./dist/py/
searx-0.15.0-py3-none-any.whl searx-0.15.0.tar.gz
To upload packages to PyPi_, there is also a ``upload-pypi`` target. It needs
twine_ to be installed. Since you are not the owner of :pypi:`searx` you will
never need the latter.

6
docs/dev/plugins.rst
View File

@ -1,7 +1,13 @@
.. _dev plugin:
=======
Plugins
=======
.. sidebar:: Further reading ..
- :ref:`plugins generic`
Plugins can extend or replace functionality of various components of searx.
Example plugin

26
docs/dev/quickstart.rst
View File

@ -4,15 +4,23 @@
Development Quickstart
======================
.. sidebar:: :ref:`makefile`
For additional developer purpose there are :ref:`makefile`.
This quickstart guide gets your environment set up with searx. Furthermore, it
gives a short introduction to the ``manage.sh`` script.
How to setup your development environment
=========================================
.. sidebar:: :ref:`make pyenv <make pyenv>`
Alternatively use the :ref:`make pyenv`.
First, clone the source code of searx to the desired folder. In this case the
source is cloned to ``~/myprojects/searx``. Then create and activate the
searx-ve virtualenv and install the required packages using manage.sh.
searx-ve virtualenv and install the required packages using ``manage.sh``.
.. code:: sh
@ -27,6 +35,10 @@ searx-ve virtualenv and install the required packages using manage.sh.
How to run tests
================
.. sidebar:: :ref:`make test.unit <make test>`
Alternatively use the ``test.pep8``, ``test.unit``, ``test.robot`` targets.
Tests can be run using the ``manage.sh`` script. Following tests and checks are
available:
@ -41,7 +53,8 @@ For example unit tests are run with the command below:
./manage.sh unit_tests
For further test options, please consult the help of the ``manage.sh`` script.
For further test options, please consult the help of the ``manage.sh`` script or
read :ref:`make test`.
How to compile styles and javascript
@ -97,6 +110,11 @@ After installing grunt, the files can be built using the following command:
Tips for debugging/development
==============================
.. sidebar:: :ref:`make run`
Makefile target ``run`` already enables debug option for your developer
session / see :ref:`make run`.
Turn on debug logging
Whether you are working on a new engine or trying to eliminate a bug, it is
always a good idea to turn on debug logging. When debug logging is enabled a
@ -104,6 +122,10 @@ Turn on debug logging
message. It can be turned on by setting ``debug: False`` to ``debug: True`` in
:origin:`settings.yml <searx/settings.yml>`.
.. sidebar:: :ref:`make test`
Alternatively use the :ref:`make test` targets.
Run ``./manage.sh tests`` before creating a PR.
Failing build on Travis is common because of PEP8 checks. So a new commit
must be created containing these format fixes. This phase can be skipped if

1428
docs/dev/reST.rst Normal file
View File

File diff suppressed because it is too large Load Diff

7
docs/dev/search_api.rst
View File

@ -14,6 +14,13 @@ Furthermore, two enpoints ``/`` and ``/search`` are available for querying.
Parameters
==========
.. sidebar:: Further reading ..
- :ref:`engines generic`
- :ref:`configured engines`
- :ref:`engine settings`
- :ref:`engine file`
``q`` : required
The search query. This string is passed to external search services. Thus,
searx supports syntax of each search service. For example, ``site:github.com

10
docs/dev/svg_image.svg Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- originate: https://commons.wikimedia.org/wiki/File:Variable_Resistor.svg -->
<svg xmlns="http://www.w3.org/2000/svg"
version="1.1" baseProfile="full"
width="70px" height="40px" viewBox="0 0 700 400">
<line x1="0" y1="200" x2="700" y2="200" stroke="black" stroke-width="20px"/>
<rect x="100" y="100" width="500" height="200" fill="white" stroke="black" stroke-width="20px"/>
<line x1="180" y1="370" x2="500" y2="50" stroke="black" stroke-width="15px"/>
<polygon points="585 0 525 25 585 50" transform="rotate(135 525 25)"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 580 B

3
requirements-dev.txt
View File

@ -11,3 +11,6 @@ transifex-client==0.12.2
unittest2==1.1.0
zope.testrunner==4.5.1
selenium==3.141.0
linuxdoc @ git+http://github.com/return42/linuxdoc.git
sphinx-jinja
sphinx-tabs

5
searx/engines/bing.py
View File

@ -89,8 +89,7 @@ def response(resp):
'content': content})
try:
result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))
result_len_container = utils.to_string(result_len_container)
result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]//text()'))
if "-" in result_len_container:
# Remove the part "from-to" for paginated request ...
result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
@ -102,7 +101,7 @@ def response(resp):
logger.debug('result error :\n%s', e)
pass
if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
if result_len and _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
return []
results.append({'number_of_results': result_len})

26
searx/engines/flickr_noapi.py
View File

@ -109,14 +109,22 @@ def response(resp):
else:
url = build_flickr_url(photo['ownerNsid'], photo['id'])
results.append({'url': url,
'title': title,
'img_src': img_src,
'thumbnail_src': thumbnail_src,
'content': content,
'author': author,
'source': source,
'img_format': img_format,
'template': 'images.html'})
result = {
'url': url,
'img_src': img_src,
'thumbnail_src': thumbnail_src,
'source': source,
'img_format': img_format,
'template': 'images.html'
}
try:
result['author'] = author.encode('utf-8')
result['title'] = title.encode('utf-8')
result['content'] = content.encode('utf-8')
except:
result['author'] = ''
result['title'] = ''
result['content'] = ''
results.append(result)
return results

9
searx/engines/ina.py
View File

@ -32,7 +32,7 @@ base_url = 'https://www.ina.fr'
search_url = base_url + '/layout/set/ajax/recherche/result?autopromote=&hf={ps}&b={start}&type=Video&r=&{query}'
# specific xpath variables
results_xpath = '//div[contains(@class,"search-results--list")]/div[@class="media"]'
results_xpath = '//div[contains(@class,"search-results--list")]//div[@class="media-body"]'
url_xpath = './/a/@href'
title_xpath = './/h3[@class="h3--title media-heading"]'
thumbnail_xpath = './/img/@src'
@ -65,8 +65,11 @@ def response(resp):
videoid = result.xpath(url_xpath)[0]
url = base_url + videoid
title = p.unescape(extract_text(result.xpath(title_xpath)))
thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
if thumbnail[0] == '/':
try:
thumbnail = extract_text(result.xpath(thumbnail_xpath)[0])
except:
thumbnail = ''
if thumbnail and thumbnail[0] == '/':
thumbnail = base_url + thumbnail
d = extract_text(result.xpath(publishedDate_xpath)[0])
d = d.split('/')

2
searx/engines/microsoft_academic.py
View File

@ -45,6 +45,8 @@ def request(query, params):
def response(resp):
results = []
response_data = loads(resp.text)
if not response_data:
return results
for result in response_data['results']:
url = _get_url(result)

2
searx/engines/scanr_structures.py
View File

@ -29,7 +29,7 @@ def request(query, params):
params['url'] = search_url
params['method'] = 'POST'
params['headers']['Content-type'] = "application/json"
params['data'] = dumps({"query": query,
params['data'] = dumps({"query": query.decode('utf-8'),
"searchField": "ALL",
"sortDirection": "ASC",
"sortOrder": "RELEVANCY",

14
searx/engines/spotify.py
View File

@ -12,10 +12,14 @@
from json import loads
from searx.url_utils import urlencode
import requests
import base64
# engine dependent config
categories = ['music']
paging = True
api_client_id = None
api_client_secret = None
# search-url
url = 'https://api.spotify.com/'
@ -31,6 +35,16 @@ def request(query, params):
params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
r = requests.post(
'https://accounts.spotify.com/api/token',
data={'grant_type': 'client_credentials'},
headers={'Authorization': 'Basic ' + base64.b64encode(
"{}:{}".format(api_client_id, api_client_secret).encode('utf-8')
).decode('utf-8')}
)
j = loads(r.text)
params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))}
return params

35
searx/settings.yml
View File

@ -79,9 +79,10 @@ engines:
categories : science
timeout : 4.0
- name : base
engine : base
shortcut : bs
# tmp suspended: dh key too small
# - name : base
# engine : base
# shortcut : bs
- name : wikipedia
engine : wikipedia
@ -408,7 +409,7 @@ engines:
- name : library genesis
engine : xpath
search_url : http://libgen.io/search.php?req={query}
search_url : https://libgen.is/search.php?req={query}
url_xpath : //a[contains(@href,"bookfi.net")]/@href
title_xpath : //a[contains(@href,"book/")]/text()[1]
content_xpath : //td/a[1][contains(@href,"=author")]/text()
@ -464,7 +465,7 @@ engines:
- name : openairedatasets
engine : json_engine
paging : True
search_url : http://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query}
search_url : https://api.openaire.eu/search/datasets?format=json&page={pageno}&size=10&title={query}
results_query : response/results/result
url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
title_query : metadata/oaf:entity/oaf:result/title/$
@ -476,7 +477,7 @@ engines:
- name : openairepublications
engine : json_engine
paging : True
search_url : http://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query}
search_url : https://api.openaire.eu/search/publications?format=json&page={pageno}&size=10&title={query}
results_query : response/results/result
url_query : metadata/oaf:entity/oaf:result/children/instance/webresource/url/$
title_query : metadata/oaf:entity/oaf:result/title/$
@ -552,10 +553,11 @@ engines:
timeout : 10.0
disabled : True
- name : scanr structures
shortcut: scs
engine : scanr_structures
disabled : True
# tmp suspended: bad certificate
# - name : scanr structures
# shortcut: scs
# engine : scanr_structures
# disabled : True
- name : soundcloud
engine : soundcloud
@ -598,9 +600,12 @@ engines:
shortcut : se
categories : science
- name : spotify
engine : spotify
shortcut : stf
# Spotify needs API credentials
# - name : spotify
# engine : spotify
# shortcut : stf
# api_client_id : *******
# api_client_secret : *******
- name : startpage
engine : startpage
@ -812,7 +817,7 @@ locales:
doi_resolvers :
oadoi.org : 'https://oadoi.org/'
doi.org : 'https://doi.org/'
doai.io : 'http://doai.io/'
sci-hub.tw : 'http://sci-hub.tw/'
doai.io : 'https://doai.io/'
sci-hub.tw : 'https://sci-hub.tw/'
default_doi_resolver : 'oadoi.org'

4
searx/settings_robot.yml
View File

@ -43,7 +43,7 @@ locales:
doi_resolvers :
oadoi.org : 'https://oadoi.org/'
doi.org : 'https://doi.org/'
doai.io : 'http://doai.io/'
sci-hub.tw : 'http://sci-hub.tw/'
doai.io : 'https://doai.io/'
sci-hub.tw : 'https://sci-hub.tw/'
default_doi_resolver : 'oadoi.org'

12
searx/static/plugins/js/vim_hotkeys.js
View File

@ -125,6 +125,14 @@ $(document).ready(function() {
}
});
function nextResult(current, direction) {
var next = current[direction]();
while (!next.is('.result') && next.length !== 0) {
next = next[direction]();
}
return next
}
function highlightResult(which) {
return function() {
var current = $('.result[data-vim-selected]');
@ -157,13 +165,13 @@ $(document).ready(function() {
}
break;
case 'down':
next = current.next('.result');
next = nextResult(current, 'next');
if (next.length === 0) {
next = $('.result:first');
}
break;
case 'up':
next = current.prev('.result');
next = nextResult(current, 'prev');
if (next.length === 0) {
next = $('.result:last');
}

1
utils/makefile.sphinx
View File

@ -198,6 +198,7 @@ $(GH_PAGES)::
$(MAKE) docs
[ -d "gh-pages/.git" ] || git clone $(GIT_URL) gh-pages
-cd $(GH_PAGES); git checkout gh-pages >/dev/null
-cd $(GH_PAGES); git pull
-cd $(GH_PAGES); ls -A | grep -v '.git$$' | xargs rm -rf
cp -r $(DOCS_DIST)/* $(GH_PAGES)/
touch $(GH_PAGES)/.nojekyll