kamp/searxng
1
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2024-11-22 12:10:11 +01:00
Code

Merge pull request #1332 from return42/searxng-install

Browse Source 
Upgrade installation scripts and documentation
This commit is contained in:
Markus Heiser 2022-07-30 13:45:12 +02:00 committed by GitHub
commit 645c2a2ca1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
62 changed files with 2261 additions and 4241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
.config.sh
View File

@ -1,52 +0,0 @@
# -*- coding: utf-8; mode: sh -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck shell=bash disable=SC2034
#
# This file should be edited only ones just before the installation of any
# service is done. After the installation of the searx service a copy of this
# file is placed into the $SEARX_SRC of the instance, e.g.::
#
# /usr/local/searx/searx-src/.config.sh
#
# .. hint::
#
# Before you change a value here, You have to fully uninstall any previous
# installation of searx, morty and filtron services!
# utils/searx.sh
# --------------
# The setup of the SearXNG instance is done in the settings.yml
# (SEARXNG_SETTINGS_PATH). Read the remarks in [1] carefully and don't forget to
# rebuild instance's environment (make buildenv) if needed. The settings.yml
# file of an already installed instance is shown by::
#
# $ ./utils/searx.sh --help
# ---- SearXNG instance setup (already installed)
# SEARXNG_SETTINGS_PATH : /etc/searxng/settings.yml
# SEARX_SRC : /usr/local/searx/searx-src
#
# [1] https://docs.searxng.org/admin/engines/settings.html
# utils/filtron.sh
# ----------------
# FILTRON_API="127.0.0.1:4005"
# FILTRON_LISTEN="127.0.0.1:4004"
# utils/morty.sh
# --------------
# morty listen address
# MORTY_LISTEN="127.0.0.1:3000"
# PUBLIC_URL_PATH_MORTY="/morty/"
# system services
# ---------------
# Common $HOME folder of the service accounts
# SERVICE_HOME_BASE="/usr/local"
# **experimental**: Set SERVICE_USER to run all services by one account, but be
# aware that removing discrete components might conflict!
# SERVICE_USER=searx

2
.github/workflows/data-update.yml vendored
View File

@ -26,7 +26,7 @@ jobs:
- name: Install Ubuntu packages
run: |
sudo ./utils/searx.sh install packages
sudo ./utils/searxng.sh install packages
- name: Set up Python
uses: actions/setup-python@v2

6
.github/workflows/integration.yml vendored
View File

@ -19,7 +19,7 @@ jobs:
uses: actions/checkout@v2
- name: Install Ubuntu packages
run: |
sudo ./utils/searx.sh install packages
sudo ./utils/searxng.sh install packages
sudo apt install firefox
- name: Set up Python
uses: actions/setup-python@v2
@ -55,7 +55,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
- name: Install Ubuntu packages
run: sudo ./utils/searx.sh install buildhost
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v2
with:
@ -82,7 +82,7 @@ jobs:
fetch-depth: '0'
persist-credentials: false
- name: Install Ubuntu packages
run: sudo ./utils/searx.sh install buildhost
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v2
with:

5
Makefile
View File

@ -59,17 +59,16 @@ test.shell:
utils/brand.env \
$(MTOOLS) \
utils/lib.sh \
utils/lib_install.sh \
utils/lib_nvm.sh \
utils/lib_static.sh \
utils/lib_go.sh \
utils/lib_redis.sh \
utils/filtron.sh \
utils/searx.sh \
utils/searxng.sh \
utils/morty.sh \
utils/lxc.sh \
utils/lxc-searx.env \
.config.sh
utils/lxc-searxng.env
$(Q)$(MTOOLS) build_msg TEST "$@ OK"

39
docs/admin/arch_public.dot
View File

@ -1,33 +1,30 @@
digraph G {
node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"];
node [style=filled, shape=box, fillcolor="#ffffcc", fontname=Sans];
edge [fontname="Sans"];
browser [label="Browser", shape=Mdiamond];
rp [label="Reverse Proxy", href="https://docs.searxng.org/utils/filtron.sh.html#public-reverse-proxy"];
filtron [label="Filtron", href="https://docs.searxng.org/utils/filtron.sh.html"];
morty [label="Morty", href="https://docs.searxng.org/utils/morty.sh.html"];
static [label="Static files", href="url to configure static files"];
uwsgi [label="uwsgi", href="https://docs.searxng.org/utils/searx.sh.html"]
searx1 [label="Searx #1"];
searx2 [label="Searx #2"];
searx3 [label="Searx #3"];
searx4 [label="Searx #4"];
browser [label="browser", shape=tab, fillcolor=aliceblue];
rp [label="reverse proxy"];
static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray];
uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searx.sh.html"]
redis [label="redis DB", shape=cylinder];
searxng1 [label="SearXNG #1", fontcolor=blue3];
searxng2 [label="SearXNG #2", fontcolor=blue3];
searxng3 [label="SearXNG #3", fontcolor=blue3];
searxng4 [label="SearXNG #4", fontcolor=blue3];
browser -> rp [label="HTTPS"]
subgraph cluster_searx {
label = "Searx instance" fontname="Sans";
subgraph cluster_searxng {
label = "SearXNG instance" fontname=Sans;
bgcolor="#fafafa";
{ rank=same; static rp };
rp -> morty [label="optional: images and HTML pages proxy"];
rp -> static [label="optional: reverse proxy serves directly static files"];
rp -> filtron [label="HTTP"];
filtron -> uwsgi [label="HTTP"];
uwsgi -> searx1;
uwsgi -> searx2;
uwsgi -> searx3;
uwsgi -> searx4;
rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray];
rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"];
uwsgi -> searxng1 -> redis;
uwsgi -> searxng2 -> redis;
uwsgi -> searxng3 -> redis;
uwsgi -> searxng4 -> redis;
}
}

20
docs/admin/architecture.rst
View File

@ -8,17 +8,19 @@ Architecture
- Reverse Proxy: :ref:`Apache <apache searxng site>` & :ref:`nginx <nginx
searxng site>`
- Filtron: :ref:`searxng filtron`
- Morty: :ref:`searxng morty`
- uWSGI: :ref:`searxng uwsgi`
- SearXNG: :ref:`installation basic`
Herein you will find some hints and suggestions about typical architectures of
SearXNG infrastructures.
We start with a contribution from :pull-searx:`@dalf <1776#issuecomment-567917320>`.
It shows a *reference* setup for public SearXNG instances which can build up and
maintained by the scripts from our :ref:`toolboxing`.
.. _architecture uWSGI:
uWSGI Setup
===========
We start with a *reference* setup for public SearXNG instances which can be build
up and maintained by the scripts from our :ref:`toolboxing`.
.. _arch public:
@ -26,3 +28,11 @@ maintained by the scripts from our :ref:`toolboxing`.
:alt: arch_public.dot
Reference architecture of a public SearXNG setup.
The reference installation activates ``server.limiter``, ``server.image_proxy``
and ``ui.static_use_hash`` (:origin:`/etc/searxng/settings.yml
<utils/templates/etc/searxng/settings.yml>`)
.. literalinclude:: ../../utils/templates/etc/searxng/settings.yml
:language: yaml
:end-before: # preferences:

8
docs/admin/buildhosts.rst
View File

@ -15,19 +15,19 @@ Buildhosts
:backlinks: entry
To get best results from build, its recommend to install additional packages
on build hosts (see :ref:`searx.sh`).::
on build hosts (see :ref:`searxng.sh`).::
sudo -H ./utils/searx.sh install buildhost
sudo -H ./utils/searxng.sh install buildhost
This will install packages needed by searx:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START distro-packages
:end-before: END distro-packages
and packages needed to build docuemtation and run tests:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START build-packages
:end-before: END build-packages

8
docs/admin/engines/nosql-engines.rst
View File

@ -42,11 +42,11 @@ Extra Dependencies
For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to
install additional packages in Python's Virtual Environment of your SearXNG
instance. To switch into the environment (:ref:`searx-src`) you can use
:ref:`searx.sh`::
instance. To switch into the environment (:ref:`searxng-src`) you can use
:ref:`searxng.sh`::
$ sudo utils/searx.sh shell
(searx-pyenv)$ pip install ...
$ sudo utils/searxng.sh instance cmd bash
(searxng-pyenv)$ pip install ...
.. _engine redis_server:

13
docs/admin/engines/settings.rst
View File

@ -207,10 +207,14 @@ Global Settings
``secret_key`` : ``$SEARXNG_SECRET``
Used for cryptography purpose.
.. _limiter:
``limiter`` :
Rate limit the number of request on the instance, block some bots. The
:ref:`limiter plugin` requires a :ref:`settings redis` database.
.. _image_proxy:
``image_proxy`` :
Allow your instance of SearXNG of being able to proxy images. Uses memory space.
@ -225,9 +229,13 @@ Global Settings
``ui:``
-------
.. _cache busting:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#caching_static_assets_with_cache_busting
.. code:: yaml
ui:
static_use_hash: false
default_locale: ""
query_in_title: false
infinite_scroll: false
@ -236,6 +244,11 @@ Global Settings
theme_args:
simple_style: auto
.. _static_use_hash:
``static_use_hash`` :
Enables `cache busting`_ of static files.
``default_locale`` :
SearXNG interface language. If blank, the locale is detected by using the
browser language. If it doesn't work, or you are deploying a language

8
docs/admin/engines/sql-engines.rst
View File

@ -98,11 +98,11 @@ Extra Dependencies
For using :ref:`engine postgresql` or :ref:`engine mysql_server` you need to
install additional packages in Python's Virtual Environment of your SearXNG
instance. To switch into the environment (:ref:`searx-src`) you can use
:ref:`searx.sh`::
instance. To switch into the environment (:ref:`searxng-src`) you can use
:ref:`searxng.sh`::
$ sudo utils/searx.sh shell
(searx-pyenv)$ pip install ...
$ sudo utils/searxng.sh instance cmd bash
(searxng-pyenv)$ pip install ...
.. _engine postgresql:

193
docs/admin/filtron.rst
View File

@ -1,193 +0,0 @@
.. _searxng filtron:
==========================
How to protect an instance
==========================
.. tip::
To protect your instance a installation of filtron (as described here) is no
longer needed, alternatively activate the :ref:`limiter plugin` in your
``settings.yml``. Note that the :ref:`limiter plugin` requires a :ref:`Redis
<settings redis>` database.
.. sidebar:: further reading
- :ref:`filtron.sh`
- :ref:`nginx searxng site`
.. _filtron: https://github.com/searxng/filtron
SearXNG depends on external search services. To avoid the abuse of these services
it is advised to limit the number of requests processed by SearXNG.
An application firewall, filtron_ solves exactly this problem. Filtron is just
a middleware between your web server (nginx, apache, ...) and searx, we describe
such infrastructures in chapter: :ref:`architecture`.
filtron & go
============
.. _Go: https://golang.org/
.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md
Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply
installed by ``go get`` package management (see `filtron README`_). If you use
filtron as middleware, a more isolated setup is recommended. To simplify such
an installation and the maintenance of, use our script :ref:`filtron.sh`.
.. _Sample configuration of filtron:
Sample configuration of filtron
===============================
.. sidebar:: Tooling box
- :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>`
An example configuration can be find below. This configuration limits the access
of:
- scripts or applications (roboagent limit)
- webcrawlers (botlimit)
- IPs which send too many requests (IP limit)
- too many json, csv, etc. requests (rss/json limit)
- the same UserAgent of if too many requests (useragent limit)
.. code:: json
[
{
"name": "search request",
"filters": [
"Param:q",
"Path=^(/|/search)$"
],
"interval": "<time-interval-in-sec (int)>",
"limit": "<max-request-number-in-interval (int)>",
"subrules": [
{
"name": "missing Accept-Language",
"filters": ["!Header:Accept-Language"],
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "suspiciously Connection=close header",
"filters": ["Header:Connection=close"],
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "IP limit",
"interval": "<time-interval-in-sec (int)>",
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"aggregations": [
"Header:X-Forwarded-For"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "rss/json limit",
"filters": [
"Param:format=(csv|json|rss)"
],
"interval": "<time-interval-in-sec (int)>",
"limit": "<max-request-number-in-interval (int)>",
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "useragent limit",
"interval": "<time-interval-in-sec (int)>",
"limit": "<max-request-number-in-interval (int)>",
"aggregations": [
"Header:User-Agent"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
}
]
}
]
.. _filtron route request:
Route request through filtron
=============================
.. sidebar:: further reading
- :ref:`filtron.sh overview`
- :ref:`installation nginx`
- :ref:`installation apache`
Filtron can be started using the following command:
.. code:: sh
$ filtron -rules rules.json
It listens on ``127.0.0.1:4004`` and forwards filtered requests to
``127.0.0.1:8888`` by default.
Use it along with ``nginx`` with the following example configuration.
.. code:: nginx
# https://example.org/searx
location /searx {
proxy_pass http://127.0.0.1:4004/;
proxy_set_header Host $host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
}
location /searx/static {
/usr/local/searx/searx-src/searx/static;
}
Requests are coming from port 4004 going through filtron and then forwarded to
port 8888 where a SearXNG is being run. For a complete setup see: :ref:`nginx
searxng site`.

6
docs/admin/index.rst
View File

@ -7,17 +7,15 @@ Administrator documentation
:caption: Contents
installation
installation-docker
installation-scripts
installation-searxng
installation-uwsgi
installation-nginx
installation-apache
installation-docker
installation-switch2ng
update-searxng
engines/index
api
architecture
filtron
morty
plugins
buildhosts

483
docs/admin/installation-apache.rst
View File

@ -1,13 +1,13 @@
.. _installation apache:
===================
Install with apache
===================
======
Apache
======
.. _Apache: https://httpd.apache.org/
.. _Apache Debian:
https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x):
.. _README.Debian:
.. _apache2.README.Debian:
https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian
.. _Apache Arch Linux:
https://wiki.archlinux.org/index.php/Apache_HTTP_Server
@ -23,7 +23,9 @@ Install with apache
https://httpd.apache.org/docs/current/en/configuring.html
.. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost
.. _LoadModule:
https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule
https://httpd.apache.org/docs/mod/mod_so.html#loadmodule
.. _IncludeOptional:
https://httpd.apache.org/docs/mod/core.html#includeoptional
.. _DocumentRoot:
https://httpd.apache.org/docs/trunk/mod/core.html#documentroot
.. _Location:
@ -32,11 +34,30 @@ Install with apache
https://uwsgi-docs.readthedocs.io/en/latest/Apache.html
.. _mod_proxy_uwsgi:
https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi
.. _mod_proxy_http:
https://httpd.apache.org/docs/current/mod/mod_proxy_http.html
.. _mod_proxy:
https://httpd.apache.org/docs/current/mod/mod_proxy.html
This section explains how to set up a SearXNG instance using the HTTP server Apache_.
If you did use the :ref:`installation scripts` and do not have any special preferences
you can install the :ref:`SearXNG site <apache searxng site>` using
:ref:`searxng.sh <searxng.sh overview>`:
.. code:: bash
$ sudo -H ./utils/searxng.sh install apache
If you have special interests or problems with setting up Apache, the following
section might give you some guidance.
.. sidebar:: further read
- `Apache Arch Linux`_
- `Apache Debian`_ and `README.Debian`_
- `Apache Debian`_
- `apache2.README.Debian`_
- `Apache Fedora`_
- `Apache directives`_
@ -45,23 +66,8 @@ Install with apache
:local:
:backlinks: entry
----
**Install** :ref:`apache searxng site` using :ref:`filtron.sh <filtron.sh overview>`
.. code:: bash
$ sudo -H ./utils/filtron.sh apache install
**Install** :ref:`apache searxng site` using :ref:`morty.sh <morty.sh overview>`
.. code:: bash
$ sudo -H ./utils/morty.sh apache install
----
The apache HTTP server
The Apache HTTP server
======================
If Apache_ is not installed, install it now. If apache_ is new to you, the
@ -73,13 +79,13 @@ Directives`_ documentation gives first orientation. There is also a list of
.. group-tab:: Ubuntu / debian
.. code:: sh
.. code:: bash
sudo -H apt-get install apache2
.. group-tab:: Arch Linux
.. code:: sh
.. code:: bash
sudo -H pacman -S apache
sudo -H systemctl enable httpd
@ -87,21 +93,21 @@ Directives`_ documentation gives first orientation. There is also a list of
.. group-tab:: Fedora / RHEL
.. code:: sh
.. code:: bash
sudo -H dnf install httpd
sudo -H systemctl enable httpd
sudo -H systemctl start httpd
Now at http://localhost you should see any kind of *Welcome* or *Test* page.
How this default intro site is configured, depends on the linux distribution
Now at http://localhost you should see some kind of *Welcome* or *Test* page.
How this default site is configured, depends on the linux distribution
(compare `Apache directives`_).
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
.. code:: bash
less /etc/apache2/sites-enabled/000-default.conf
@ -115,7 +121,7 @@ How this default intro site is configured, depends on the linux distribution
.. group-tab:: Arch Linux
.. code:: sh
.. code:: bash
less /etc/httpd/conf/httpd.conf
@ -130,8 +136,8 @@ How this default intro site is configured, depends on the linux distribution
Require all granted
</Directory>
The *welcome* page of Arch Linux is a page showing directory located at
``DocumentRoot``. This is *directory* page is generated by the Module
The *welcome* page of Arch Linux is a page showing the directory located
at ``DocumentRoot``. This *directory* page is generated by the Module
`mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_:
.. code:: apache
@ -142,7 +148,7 @@ How this default intro site is configured, depends on the linux distribution
.. group-tab:: Fedora / RHEL
.. code:: sh
.. code:: bash
less /etc/httpd/conf/httpd.conf
@ -163,323 +169,204 @@ How this default intro site is configured, depends on the linux distribution
less /etc/httpd/conf.d/welcome.conf
.. _apache searxng site:
Apache Reverse Proxy
====================
.. _Debian's Apache layout:
.. sidebar:: public to the internet?
Debian's Apache layout
----------------------
If your SearXNG instance is public, stop here and first install :ref:`filtron
reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
:ref:`installation scripts`. If already done, follow setup: *SearXNG via
filtron plus morty*.
Be aware, Debian's Apache layout is quite different from the standard Apache
configuration. For details look at the apache2.README.Debian_
(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on
Debian:
To setup a Apache revers proxy you have to enable the *headers* and *proxy*
modules and create a `Location`_ configuration for the SearXNG site. In most
distributions you have to un-comment the lines in the main configuration file,
except in :ref:`The Debian Layout`.
* :man:`apache2ctl`: Apache HTTP server control interface
* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules
* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations
* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites
.. _apache modules:
Apache modules
--------------
To load additional modules, in most distributions you have to un-comment the
lines with the corresponding LoadModule_ directive, except in :ref:`Debian's
Apache layout`.
.. tabs::
.. group-tab:: Ubuntu / debian
In the Apache setup, enable headers and proxy modules:
:ref:`Debian's Apache layout` uses :man:`a2enmod` and :man:`a2dismod` to
activate or disable modules:
.. code:: sh
.. code:: bash
sudo -H a2enmod ssl
sudo -H a2enmod headers
sudo -H a2enmod proxy
sudo -H a2enmod proxy_http
sudo -H a2enmod proxy_uwsgi
In :ref:`The Debian Layout` you create a ``searxng.conf`` with the
``<Location /searx >`` directive and save this file in the *sites
.. group-tab:: Arch Linux
In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_
directives:
.. code:: apache
LoadModule ssl_module modules/mod_ssl.so
LoadModule headers_module modules/mod_headers.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
.. group-tab:: Fedora / RHEL
In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_
directives:
.. code:: apache
LoadModule ssl_module modules/mod_ssl.so
LoadModule headers_module modules/mod_headers.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
.. _apache sites:
Apache sites
------------
.. tabs::
.. group-tab:: Ubuntu / debian
In :ref:`Debian's Apache layout` you create a ``searxng.conf`` with the
``<Location /searxng >`` directive and save this file in the *sites
available* folder at ``/etc/apache2/sites-available``. To enable the
``searxng.conf`` use :man:`a2ensite`:
.. code:: sh
.. code:: bash
sudo -H a2ensite searxng.conf
.. group-tab:: Arch Linux
In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
modules (LoadModule_):
In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_
directive:
.. code:: apache
FIXME needs test
IncludeOptional sites-enabled/*.conf
LoadModule headers_module modules/mod_headers.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
.. group-tab:: Fedora / RHEL
In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy
modules (LoadModule_):
.. code:: apache
FIXME needs test
LoadModule headers_module modules/mod_headers.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the
proxied host.
.. _apache searxng via filtron plus morty:
.. tabs::
.. group-tab:: SearXNG via filtron plus morty
Use this setup, if your instance is public to the internet, compare
figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
*localhost 4004* (:ref:`filtron route request`):
.. code:: apache
<Location /searx >
# SetEnvIf Request_URI "/searx" dontlog
# CustomLog /dev/null combined env=dontlog
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass http://127.0.0.1:4004
RequestHeader set X-Script-Name /searx
</Location>
2. Configure reverse proxy for :ref:`morty <searxng morty>`, listening on
*localhost 3000*
.. code:: apache
ProxyPreserveHost On
<Location /morty >
# SetEnvIf Request_URI "/morty" dontlog
# CustomLog /dev/null combined env=dontlog
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPass http://127.0.0.1:3000
RequestHeader set X-Script-Name /morty
</Location>
For a fully result proxification add :ref:`morty's <searxng morty>` **public
URL** to your :origin:`searx/settings.yml`:
.. code:: yaml
result_proxy:
# replace example.org with your server's public name
url : https://example.org/morty
key : !!binary "insert_your_morty_proxy_key_here"
server:
image_proxy : True
uWSGI support
=============
Be warned, with this setup, your instance isn't :ref:`protected <searxng
filtron>`, nevertheless it is good enough for intranet usage. In modern Linux
distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache
package and you need to install only the :ref:`uWSGI <searxng uwsgi>` package:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
sudo -H apt-get install uwsgi
# Ubuntu =< 18.04
sudo -H apt-get install libapache2-mod-proxy-uwsgi
.. group-tab:: Arch Linux
.. code:: sh
sudo -H pacman -S uwsgi
.. group-tab:: Fedora / RHEL
.. code:: sh
sudo -H dnf install uwsgi
The next example shows a configuration using the `uWSGI Apache support`_ via
unix sockets and `mod_proxy_uwsgi`_.
For socket communication, you have to activate ``socket =
/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888``
configuration in your :ref:`uwsgi ini file <uwsgi configuration>`. If not
already exists, create a folder for the unix sockets, which can be used by the
SearXNG account (see :ref:`create searxng user`):
Create two folders, one for the *available sites* and one for the *enabled sites*:
.. code:: bash
sudo -H mkdir -p /run/uwsgi/app/searx/
sudo -H chown -R searx:searx /run/uwsgi/app/searx/
mkdir -p /etc/httpd/sites-available
mkdir -p /etc/httpd/sites-enabled
If the server is public; to limit access to your intranet replace ``Allow from
all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class.
Create configuration at ``/etc/httpd/sites-available`` and place a
symlink to ``sites-enabled``:
.. tabs::
.. code:: bash
.. group-tab:: Ubuntu / debian
.. code:: apache
LoadModule headers_module /usr/lib/apache2/mod_headers.so
LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so
# SetEnvIf Request_URI /searx dontlog
# CustomLog /dev/null combined env=dontlog
<Location /searx>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
</Location>
.. group-tab:: Arch Linux
.. code:: apache
FIXME needs test
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
# SetEnvIf Request_URI /searx dontlog
# CustomLog /dev/null combined env=dontlog
<Location /searx>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
</Location>
sudo -H ln -s /etc/httpd/sites-available/searxng.conf \
/etc/httpd/sites-enabled/searxng.conf
.. group-tab:: Fedora / RHEL
.. code:: apache
FIXME needs test
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so
<IfModule proxy_uwsgi_module>
# SetEnvIf Request_URI /searx dontlog
# CustomLog /dev/null combined env=dontlog
<Location /searx>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/
</Location>
</IfModule>
.. group-tab:: old mod_wsgi
We show this only for historical reasons, DON'T USE `mod_uwsgi
<https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi>`_.
ANYMORE!
In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_
directive:
.. code:: apache
<IfModule mod_uwsgi.c>
IncludeOptional sites-enabled/*.conf
# SetEnvIf Request_URI "/searx" dontlog
# CustomLog /dev/null combined env=dontlog
Create two folders, one for the *available sites* and one for the *enabled sites*:
<Location /searx >
.. code:: bash
Require all granted
mkdir -p /etc/httpd/sites-available
mkdir -p /etc/httpd/sites-enabled
Options FollowSymLinks Indexes
SetHandler uwsgi-handler
uWSGISocket /run/uwsgi/app/searx/socket
Create configuration at ``/etc/httpd/sites-available`` and place a
symlink to ``sites-enabled``:
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
.. code:: bash
</Location>
sudo -H ln -s /etc/httpd/sites-available/searxng.conf \
/etc/httpd/sites-enabled/searxng.conf
</IfModule>
.. _apache searxng site:
Apache's SearXNG site
=====================
.. _mod_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi
.. sidebar:: uWSGI
Use mod_proxy_uwsgi_ / don't use the old mod_uwsgi_ anymore.
To proxy the incoming requests to the SearXNG instance Apache needs the
mod_proxy_ module (:ref:`apache modules`).
.. sidebar:: HTTP headers
With ProxyPreserveHost_ the incoming ``Host`` header is passed to the proxied
host.
Depending on what your SearXNG installation is listening on, you need a http
mod_proxy_http_) or socket (mod_proxy_uwsgi_) communication to upstream.
The :ref:`installation scripts` installs the :ref:`reference setup
<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default.
You can install and activate your own ``searxng.conf`` like shown in
:ref:`apache sites`.
.. tabs::
.. group-tab:: socket
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START apache socket
:end-before: END apache socket
.. group-tab:: http
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START apache http
:end-before: END apache http
.. _restart apache:
Restart service
===============
Restart service:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
.. code:: bash
sudo -H systemctl restart apache2
sudo -H service uwsgi restart searx
sudo -H service uwsgi restart searxng
.. group-tab:: Arch Linux
.. code:: sh
.. code:: bash
sudo -H systemctl restart httpd
sudo -H systemctl restart uwsgi@searx
sudo -H systemctl restart uwsgi@searxng
.. group-tab:: Fedora / RHEL
.. code:: sh
.. code:: bash
sudo -H systemctl restart httpd
sudo -H touch /etc/uwsgi.d/searxng.ini
@ -489,27 +376,13 @@ disable logs
============
For better privacy you can disable Apache logs. In the examples above activate
one of the lines and `restart apache`_::
one of the lines and `restart apache`_:
.. code:: apache
# SetEnvIf Request_URI "/searx" dontlog
SetEnvIf Request_URI "/searxng" dontlog
# CustomLog /dev/null combined env=dontlog
The ``CustomLog`` directive disable logs for the whole (virtual) server, use it
when the URL of the service does not have a path component (``/searx``) / is
located at root (``/``).
.. _The Debian Layout:
The Debian Layout
=================
Be aware that the Debian layout is quite different from the standard Apache
configuration. For details look at the README.Debian_
(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on
Debian:
* :man:`apache2ctl`: Apache HTTP server control interface
* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules
* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations
* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites
The ``CustomLog`` directive disables logs for the entire (virtual) server, use it
when the URL of the service does not have a path component (``/searxng``), so when
SearXNG is located at root (``/``).

59
docs/admin/installation-docker.rst
View File

@ -1,37 +1,60 @@
.. _installation docker:
===================
Docker installation
===================
================
Docker Container
================
.. _ENTRYPOINT: https://docs.docker.com/engine/reference/builder/#entrypoint
.. _searxng/searxng @dockerhub: https://hub.docker.com/r/searxng/searxng
.. _searxng-docker: https://github.com/searxng/searxng-docker
.. _[filtron]: https://hub.docker.com/r/dalf/filtron
.. _[morty]: https://hub.docker.com/r/dalf/morty
.. _[caddy]: https://hub.docker.com/_/caddy
.. _Redis: https://redis.io/
----
.. sidebar:: info
- `searxng/searxng @dockerhub`_
- :origin:`Dockerfile`
- `searxng/searxng @dockerhub <https://hub.docker.com/r/searxng/searxng>`_
- `Docker overview <https://docs.docker.com/get-started/overview>`_
- `Docker Cheat Sheet <https://www.docker.com/sites/default/files/d8/2019-09/docker-cheat-sheet.pdf>`_
- `Alpine Linux <https://alpinelinux.org>`_ `(wiki) <https://en.wikipedia.org/wiki/Alpine_Linux>`__ `apt packages <https://pkgs.alpinelinux.org/packages>`_
- `Docker Cheat Sheet <https://docs.docker.com/get-started/docker_cheatsheet.pdf>`_
- `Alpine Linux <https://alpinelinux.org>`_
`(wiki) <https://en.wikipedia.org/wiki/Alpine_Linux>`__
`apt packages <https://pkgs.alpinelinux.org/packages>`_
- Alpine's ``/bin/sh`` is :man:`dash`
.. tip::
**If you intend to create a public instance using Docker, use our well maintained
docker container**
If you intend to create a public instance using Docker, use our well
maintained searxng-docker_ image which includes
- `searxng/searxng @dockerhub`_.
- :ref:`protection <searxng filtron>` `[filtron]`_,
- a :ref:`result proxy <searxng morty>` `[morty]`_ and
- a HTTPS reverse proxy `[caddy]`_.
.. sidebar:: hint
Make sure you have `installed Docker <https://docs.docker.com/get-docker/>`_ and
on Linux, don't forget to add your user to the docker group (log out and log
back in so that your group membership is re-evaluated):
The rest of this article is of interest only to those who want to create and
maintain their own Docker images.
The sources are hosted at searxng-docker_ and the container includes:
- a HTTPS reverse proxy `[caddy]`_ and
- a Redis_ DB
The `default SearXNG setup <https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml>`_
of this container:
- enables :ref:`limiter <limiter>` to protect against bots
- enables :ref:`image proxy <image_proxy>` for better privacy
- enables :ref:`cache busting <static_use_hash>` to save bandwith
----
Get Docker
==========
If you plan to build and maintain a docker image by yourself, make sure you have
`Docker installed <https://docs.docker.com/get-docker/>`_. On Linux don't
forget to add your user to the docker group (log out and log back in so that
your group membership is re-evaluated):
.. code:: sh

297
docs/admin/installation-nginx.rst
View File

@ -1,8 +1,8 @@
.. _installation nginx:
==================
Install with nginx
==================
=====
NGINX
=====
.. _nginx:
https://docs.nginx.com/nginx/admin-guide/
@ -19,6 +19,19 @@ Install with nginx
.. _SCRIPT_NAME:
https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name
This section explains how to set up a SearXNG instance using the HTTP server nginx_.
If you have used the :ref:`installation scripts` and do not have any special preferences
you can install the :ref:`SearXNG site <nginx searxng site>` using
:ref:`searxng.sh <searxng.sh overview>`:
.. code:: bash
$ sudo -H ./utils/searxng.sh install nginx
If you have special interests or problems with setting up nginx, the following
section might give you some guidance.
.. sidebar:: further reading
- nginx_
@ -27,39 +40,23 @@ Install with nginx
- `Getting Started wiki`_
- `uWSGI support from nginx`_
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
----
**Install** :ref:`nginx searxng site` using :ref:`filtron.sh <filtron.sh overview>`
.. code:: bash
$ sudo -H ./utils/filtron.sh nginx install
**Install** :ref:`nginx searxng site` using :ref:`morty.sh <morty.sh overview>`
.. code:: bash
$ sudo -H ./utils/morty.sh nginx install
----
The nginx HTTP server
=====================
If nginx_ is not installed (uwsgi will not work with the package nginx-light),
install it now.
If nginx_ is not installed, install it now.
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
.. code:: bash
sudo -H apt-get install nginx
@ -81,18 +78,18 @@ install it now.
Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you
see a *Fedora Webserver - Test Page*. The test page comes from the default
`nginx server configuration`_. How this default intro site is configured,
`nginx server configuration`_. How this default site is configured,
depends on the linux distribution:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
.. code:: bash
less /etc/nginx/nginx.conf
there is a line including site configurations from:
There is one line that includes site configurations from:
.. code:: nginx
@ -104,7 +101,7 @@ depends on the linux distribution:
less /etc/nginx/nginx.conf
in there is a configuration section named ``server``:
There is a configuration section named ``server``:
.. code-block:: nginx
@ -120,249 +117,121 @@ depends on the linux distribution:
less /etc/nginx/nginx.conf
there is a line including site configurations from:
There is one line that includes site configurations from:
.. code:: nginx
include /etc/nginx/conf.d/*.conf;
.. _nginx searxng site:
A nginx SearXNG site
NGINX's SearXNG site
====================
.. sidebar:: public to the internet?
Now you have to create a configuration file (``searxng.conf``) for the SearXNG
site. If nginx_ is new to you, the `nginx beginners guide`_ is a good starting
point and the `Getting Started wiki`_ is always a good resource *to keep in the
pocket*.
If your SearXNG instance is public, stop here and first install :ref:`filtron
reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see
:ref:`installation scripts`. If already done, follow setup: *SearXNG via
filtron plus morty*.
Depending on what your SearXNG installation is listening on, you need a http or socket
communication to upstream.
Now you have to create a configuration for the SearXNG site. If nginx_ is new to
you, the `nginx beginners guide`_ is a good starting point and the `Getting
Started wiki`_ is always a good resource *to keep in the pocket*.
.. tabs::
.. group-tab:: socket
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START nginx socket
:end-before: END nginx socket
.. group-tab:: http
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START nginx http
:end-before: END nginx http
The :ref:`installation scripts` installs the :ref:`reference setup
<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default.
.. tabs::
.. group-tab:: Ubuntu / debian
Create configuration at ``/etc/nginx/sites-available/searxng`` and place a
symlink to sites-enabled:
Create configuration at ``/etc/nginx/sites-available/`` and place a
symlink to ``sites-enabled``:
.. code:: sh
.. code:: bash
sudo -H ln -s /etc/nginx/sites-available/searxng /etc/nginx/sites-enabled/searxng
sudo -H ln -s /etc/nginx/sites-available/searxng.conf \
/etc/nginx/sites-enabled/searxng.conf
.. group-tab:: Arch Linux
In the ``/etc/nginx/nginx.conf`` file, replace the configuration section
named ``server``.
.. group-tab:: Fedora / RHEL
Create configuration at ``/etc/nginx/conf.d/searxng`` and place a
symlink to sites-enabled:
.. _nginx searxng via filtron plus morty:
.. tabs::
.. group-tab:: SearXNG via filtron plus morty
Use this setup, if your instance is public to the internet, compare
figure: :ref:`architecture <arch public>` and :ref:`installation scripts`.
1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on
*localhost 4004* (:ref:`filtron route request`):
.. code:: nginx
# https://example.org/searx
location /searx {
proxy_pass http://127.0.0.1:4004/;
proxy_set_header Host $host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
}
location /searx/static/ {
alias /usr/local/searx/searx-src/searx/static/;
}
2. Configure reverse proxy for :ref:`morty <searxng morty>`, listening on
*localhost 3000*:
.. code:: nginx
# https://example.org/morty
location /morty {
proxy_pass http://127.0.0.1:3000/;
proxy_set_header Host $host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
}
For a fully result proxification add :ref:`morty's <searxng morty>` **public
URL** to your :origin:`searx/settings.yml`:
.. code:: yaml
result_proxy:
# replace example.org with your server's public name
url : https://example.org/morty
key : !!binary "insert_your_morty_proxy_key_here"
server:
image_proxy : True
.. group-tab:: proxy or uWSGI
Be warned, with this setup, your instance isn't :ref:`protected <searxng
filtron>`. Nevertheless it is good enough for intranet usage and it is a
excellent example of; *how different services can be set up*. The next
example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI
application <uwsgi configuration>`, listening on ``http =
127.0.0.1:8888``.
.. code:: nginx
# https://hostname.local/
location / {
proxy_pass http://127.0.0.1:8888;
proxy_set_header Host $host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_buffering off;
}
Alternatively you can use the `uWSGI support from nginx`_ via unix
sockets. For socket communication, you have to activate ``socket =
/run/uwsgi/app/searx/socket`` and comment out the ``http =
127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file <uwsgi
configuration>`.
The example shows a nginx virtual ``server`` configuration, listening on
port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at
location ``/`` by importing the `uwsgi_params`_ and passing requests to
the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the
:ref:`searx-src clone <searx-src>` and wraps directly the
:origin:`searx/static/` content at ``location /static``.
In the ``/etc/nginx/nginx.conf`` file, in the ``server`` section add a
`include <https://nginx.org/en/docs/ngx_core_module.html#include>`_
directive:
.. code:: nginx
server {
# replace hostname.local with your server's name
server_name hostname.local;
listen 80;
listen [::]:80;
location / {
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
# ...
include /etc/nginx/default.d/*.conf;
# ...
}
root /usr/local/searx/searx-src/searx;
location /static { }
}
If not already exists, create a folder for the unix sockets, which can be
used by the SearXNG account:
Create two folders, one for the *available sites* and one for the *enabled sites*:
.. code:: bash
mkdir -p /run/uwsgi/app/searx/
sudo -H chown -R searx:searx /run/uwsgi/app/searx/
mkdir -p /etc/nginx/default.d
mkdir -p /etc/nginx/default.apps-available
.. group-tab:: \.\. at subdir URL
Create configuration at ``/etc/nginx/default.apps-available`` and place a
symlink to ``default.d``:
Be warned, with these setups, your instance isn't :ref:`protected <searxng
filtron>`. The examples are just here to demonstrate how to export the
SearXNG application from a subdirectory URL ``https://example.org/searx/``.
.. code:: bash
.. code:: nginx
sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \
/etc/nginx/default.d/searxng.conf
# https://hostname.local/searx
.. group-tab:: Fedora / RHEL
location /searx {
proxy_pass http://127.0.0.1:8888;
Create a folder for the *available sites*:
proxy_set_header Host $host;
proxy_set_header Connection $http_connection;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Scheme $scheme;
proxy_set_header X-Script-Name /searx;
proxy_buffering off;
}
.. code:: bash
location /searx/static/ {
alias /usr/local/searx/searx-src/searx/static/;
}
mkdir -p /etc/nginx/default.apps-available
The ``X-Script-Name /searx`` is needed by the SearXNG implementation to
calculate relative URLs correct. The next example shows a uWSGI
configuration. Since there are no HTTP headers in a (u)WSGI protocol, the
value is shipped via the SCRIPT_NAME_ in the WSGI environment.
Create configuration at ``/etc/nginx/default.apps-available`` and place a
symlink to ``conf.d``:
.. code:: nginx
.. code:: bash
# https://hostname.local/searx
sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \
/etc/nginx/conf.d/searxng.conf
location /searx {
uwsgi_param SCRIPT_NAME /searx;
include uwsgi_params;
uwsgi_pass unix:/run/uwsgi/app/searx/socket;
}
location /searx/static/ {
alias /usr/local/searx/searx-src/searx/;
}
For SearXNG to work correctly the ``base_url`` must be set in the
:origin:`searx/settings.yml`.
.. code:: yaml
server:
# replace example.org with your server's public name
base_url : https://example.org/searx/
Restart service:
Restart services:
.. tabs::
.. group-tab:: Ubuntu / debian
.. code:: sh
.. code:: bash
sudo -H systemctl restart nginx
sudo -H service uwsgi restart searx
sudo -H service uwsgi restart searxng
.. group-tab:: Arch Linux
.. code:: sh
.. code:: bash
sudo -H systemctl restart nginx
sudo -H systemctl restart uwsgi@searx
sudo -H systemctl restart uwsgi@searxng
.. group-tab:: Fedora
.. group-tab:: Fedora / RHEL
.. code:: sh
.. code:: bash
sudo -H systemctl restart nginx
sudo -H touch /etc/uwsgi.d/searxng.ini

62
docs/admin/installation-scripts.rst Normal file
View File

@ -0,0 +1,62 @@
.. _installation scripts:
===================
Installation Script
===================
.. sidebar:: Update the OS first!
To avoid unwanted side effects, update your OS before installing SearXNG.
The following will install a setup as shown in :ref:`the reference architecture
<arch public>`. First you need to get a clone of the repository. The clone is only needed for
the installation procedure and some maintenance tasks.
.. sidebar:: further read
- :ref:`toolboxing`
Jump to a folder that is readable by *others* and start to clone SearXNG,
alternatively you can create your own fork and clone from there.
.. code:: bash
$ cd ~/Downloads
$ git clone https://github.com/searxng/searxng.git searxng
$ cd searxng
.. sidebar:: further read
- :ref:`inspect searxng`
To install a SearXNG :ref:`reference setup <use_default_settings.yml>`
including a :ref:`uWSGI setup <architecture uWSGI>` as described in the
:ref:`installation basic` and in the :ref:`searxng uwsgi` section type:
.. code:: bash
$ sudo -H ./utils/searxng.sh install all
.. attention::
For the installation procedure, use a *sudoer* login to run the scripts. If
you install from ``root``, take into account that the scripts are creating a
``searxng`` user. In the installation procedure this new created user does
need read access to the cloned SearXNG repository, which is not the case if you clone
it into a folder below ``/root``!
.. sidebar:: further read
- :ref:`update searxng`
.. _caddy: https://hub.docker.com/_/caddy
When all services are installed and running fine, you can add SearXNG to your
HTTP server. We do not have any preferences for the HTTP server, you can use
whatever you prefer.
We use caddy in our :ref:`docker image <installation docker>` and we have
implemented installation procedures for:
- :ref:`installation nginx`
- :ref:`installation apache`

56
docs/admin/installation-searxng.rst
View File

@ -9,15 +9,16 @@ Step by step installation
:local:
:backlinks: entry
Step by step installation with virtualenv. For Ubuntu, be sure to have enable
universe repository.
In this section we show the setup of a SearXNG instance that will be installed
by the :ref:`installation scripts`.
.. _install packages:
Install packages
================
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START distro-packages
:end-before: END distro-packages
@ -30,32 +31,32 @@ Install packages
Create user
===========
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START create user
:end-before: END create user
.. _searx-src:
.. _searxng-src:
Install SearXNG & dependencies
==============================
Start a interactive shell from new created user and clone searx:
Start a interactive shell from new created user and clone SearXNG:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START clone searxng
:end-before: END clone searxng
In the same shell create *virtualenv*:
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START create virtualenv
:end-before: END create virtualenv
To install searx's dependencies, exit the SearXNG *bash* session you opened above
and restart a new. Before install, first check if your *virtualenv* was sourced
To install SearXNG's dependencies, exit the SearXNG *bash* session you opened above
and start a new one. Before installing, check if your *virtualenv* was sourced
from the login (*~/.profile*):
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START manage.sh update_packages
:end-before: END manage.sh update_packages
@ -77,18 +78,17 @@ Configuration
- :ref:`settings use_default_settings`
- :origin:`/etc/searxng/settings.yml <utils/templates/etc/searxng/settings.yml>`
To create a initial ``/etc/searxng/settings.yml`` you can start with a copy of
the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup
To create a initial ``/etc/searxng/settings.yml`` we recommend to start with a
copy of the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup
:ref:`use default settings <settings use_default_settings>` from
:origin:`searx/settings.yml`.
:origin:`searx/settings.yml` and is shown in the tab *"Use default settings"*
below. This setup:
For a *minimal setup*, configure like shown below – replace ``searx@$(uname
-n)`` with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit
``/etc/searxng/settings.yml`` to your needs.
- enables :ref:`limiter <limiter>` to protect against bots
- enables :ref:`image proxy <image_proxy>` for better privacy
- enables :ref:`cache busting <static_use_hash>` to save bandwith
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
:start-after: START searxng config
:end-before: END searxng config
Modify the ``/etc/searxng/settings.yml`` to your needs:
.. tabs::
@ -96,11 +96,23 @@ For a *minimal setup*, configure like shown below – replace ``searx@$(uname
.. literalinclude:: ../../utils/templates/etc/searxng/settings.yml
:language: yaml
:end-before: # hostname_replace:
To see the entire file jump to :origin:`utils/templates/etc/searxng/settings.yml`
.. group-tab:: searx/settings.yml
.. literalinclude:: ../../searx/settings.yml
:language: yaml
:end-before: # hostname_replace:
To see the entire file jump to :origin:`searx/settings.yml`
For a *minimal setup* you need to set ``server:secret_key``.
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START searxng config
:end-before: END searxng config
Check
@ -110,11 +122,11 @@ To check your SearXNG setup, optional enable debugging and start the *webapp*.
SearXNG looks at the exported environment ``$SEARXNG_SETTINGS_PATH`` for a
configuration file.
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START check searxng installation
:end-before: END check searxng installation
If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the
debug option in ``settings.yml``. You can now exit SearXNG user bash (enter exit
debug option in ``settings.yml``. You can now exit SearXNG user bash session (enter exit
command twice). At this point SearXNG is not demonized; uwsgi allows this.

75
docs/admin/installation-switch2ng.rst
View File

@ -1,75 +0,0 @@
.. _installation switch2ng:
============================
Switch from searx to SearXNG
============================
.. sidebar:: info
- :pull:`456`
- :pull:`A comment about rolling release <446#issuecomment-954730358>`
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
If you have a searx installation on your sever and want to switch to SearXNG,
you need to uninstall searx first. If you have an old searx docker installation
replace your docker image / see :ref:`installation docker`.
If your searx instance was installed *"Step by step"* or by the *"Installation
scripts"*, you need to undo the installation procedure completely. If you have
morty & filtron installed, it is recommended to uninstall these services also.
In case of scripts, to uninstall use the scripts from the origin you installed
searx from.
If you have removed the old searx installation, clone from SearXNG and and start
with your installation procedure (e.g. :ref:`installation scripts`):
.. code:: bash
$ cd ~/Downloads
$ git clone https://github.com/searxng/searxng.git searxng
$ cd searxng
$ ...
``.config.sh``
==============
Please take into account; SearXNG has normalized ``.config.sh`` with
``settings.yml`` and some of the environment settings has been removed from or
renamed in the ``.config.sh``:
- :patch:`[mod] normalize .config.sh with settings.yml <f61c918d>`
- :patch:`[fix] ./utils/filtron.sh - FILTRON_TARGET from YAML settings <7196a9b5>`
- :patch:`SearXNG: SEARXNG_SETTINGS_PATH <253b8503>`
Check after Installation
========================
Once you have done your installation, you can run a SearXNG *check* procedure,
to see if there are some left overs. In this example there exists a *old*
``/etc/searx/settings.yml``::
$ sudo -H ./utils/searx.sh install check
============================
SearXNG (check installation)
============================
ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/
INFO: SearXNG instance already installed at: /usr/local/searx/searx-src
...
INFO: Service account searx exists.
INFO: ~searx: python environment is available.
INFO: ~searx: SearXNG software is installed.
INFO: uWSGI app searxng.ini is enabled.
INFO searx : merge the default settings ( /usr/local/searx/searx-src/searx/settings.yml ) and the user setttings ( /etc/searxng/settings.yml )
INFO searx : max_request_timeout=None
To *check* the filtron & morty installations, use similar commands::
$ sudo -H /utils/filtron.sh install check
$ sudo -H /utils/morty.sh install check

204
docs/admin/installation-uwsgi.rst
View File

@ -1,7 +1,7 @@
.. _searxng uwsgi:
=====
uwsgi
uWSGI
=====
.. sidebar:: further reading
@ -29,51 +29,77 @@ uwsgi
Origin uWSGI
============
How uWSGI is implemented by distributors is different. uWSGI itself
recommend two methods
.. _Tyrant mode:
https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting
`systemd.unit`_ template files as described here `One service per app in systemd`_.
How uWSGI is implemented by distributors varies. The uWSGI project itself
recommends two methods:
There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app
placed at dedicated locations. Take archlinux and a searxng.ini as example::
1. `systemd.unit`_ template file as described here `One service per app in systemd`_:
unit template --> /usr/lib/systemd/system/uwsgi@.service
uwsgi ini files --> /etc/uwsgi/searxng.ini
There is one `systemd unit template`_ on the system installed and one `uwsgi
ini file`_ per uWSGI-app placed at dedicated locations. Take archlinux and a
``searxng.ini`` as example::
The SearXNG app can be maintained as know from common systemd units::
systemd template unit: /usr/lib/systemd/system/uwsgi@.service
contains: [Service]
ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/%I.ini
systemctl enable uwsgi@searx
systemctl start uwsgi@searx
systemctl restart uwsgi@searx
systemctl stop uwsgi@searx
SearXNG application: /etc/uwsgi/searxng.ini
links to: /etc/uwsgi/apps-available/searxng.ini
The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps.
The SearXNG app (template ``/etc/uwsgi/%I.ini``) can be maintained as known
from common systemd units:
.. code:: sh
$ systemctl enable uwsgi@searxng
$ systemctl start uwsgi@searxng
$ systemctl restart uwsgi@searxng
$ systemctl stop uwsgi@searxng
2. The `uWSGI Emperor`_ which fits for maintaining a large range of uwsgi
apps and there is a `Tyrant mode`_ to secure multi-user hosting.
The Emperor mode is a special uWSGI instance that will monitor specific
events. The Emperor mode (service) is started by a (common, not template)
systemd unit. The Emperor service will scan specific directories for `uwsgi
ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the
timestamp is modified, a corresponding action takes place: a new uWSGI
instance is started, reload or stopped. Take Fedora and a searxng.ini as
example::
events. The Emperor mode (the service) is started by a (common, not template)
systemd unit.
to start a new SearXNG instance create --> /etc/uwsgi.d/searxng.ini
The Emperor service will scan specific directories for `uwsgi ini file`_\s
(also know as *vassals*). If a *vassal* is added, removed or the timestamp is
modified, a corresponding action takes place: a new uWSGI instance is started,
reload or stopped. Take Fedora and a ``searxng.ini`` as example::
to install & start SearXNG instance create --> /etc/uwsgi.d/searxng.ini
to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini
to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini
Distributors
============
The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors
mostly offer their users, even if they differ in the way they implement both
modes and their defaults. Another point they might differ is the packaging of
modes and their defaults. Another point they might differ in is the packaging of
plugins (if so, compare :ref:`install packages`) and what the default python
interpreter is (python2 vs. python3).
Fedora starts a Emperor by default, while archlinux does not start any uwsgi
service by default. Worth to know; debian (ubuntu) follow a complete different
approach. *debian*: your are familiar with the apache infrastructure? .. they
do similar for the uWSGI infrastructure (with less comfort), the folders are::
While archlinux does not start a uWSGI service by default, Fedora (RHEL) starts
a Emperor in `Tyrant mode`_ by default (you should have read :ref:`uWSGI Tyrant
mode pitfalls`). Worth to know; debian (ubuntu) follow a complete different
approach, read see :ref:`Debian's uWSGI layout`.
.. _Debian's uWSGI layout:
Debian's uWSGI layout
---------------------
.. _uwsgi.README.Debian:
https://salsa.debian.org/uwsgi-team/uwsgi/-/raw/debian/latest/debian/uwsgi.README.Debian
Be aware, Debian's uWSGI layout is quite different from the standard uWSGI
configuration. Your are familiar with :ref:`Debian's Apache layout`? .. they do a
similar thing for the uWSGI infrastructure. The folders are::
/etc/uwsgi/apps-available/
/etc/uwsgi/apps-enabled/
@ -82,29 +108,52 @@ The `uwsgi ini file`_ is enabled by a symbolic link::
ln -s /etc/uwsgi/apps-available/searxng.ini /etc/uwsgi/apps-enabled/
From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You
could control specific instance(s) by issuing::
More details can be found in the uwsgi.README.Debian_
(``/usr/share/doc/uwsgi/README.Debian.gz``). Some commands you should know on
Debian:
service uwsgi <command> <confname> <confname> ...
.. code:: none
sudo -H service uwsgi start searx
sudo -H service uwsgi stop searx
Commands recognized by init.d script
====================================
My experience is, that this command is a bit buggy.
You can issue to init.d script following commands:
* start | starts daemon
* stop | stops daemon
* reload | sends to daemon SIGHUP signal
* force-reload | sends to daemon SIGTERM signal
* restart | issues 'stop', then 'start' commands
* status | shows status of daemon instance (running/not running)
.. _uwsgi configuration:
'status' command must be issued with exactly one argument: '<confname>'.
Alltogether
===========
Controlling specific instances of uWSGI
=======================================
Create the configuration ini-file according to your distribution (see below) and
restart the uwsgi application.
You could control specific instance(s) by issuing:
SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi <command> <confname> <confname>...
where:
* <command> is one of 'start', 'stop' etc.
* <confname> is the name of configuration file (without extension)
For example, this is how instance for /etc/uwsgi/apps-enabled/hello.xml is
started:
SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi start hello
.. _uWSGI maintenance:
uWSGI maintenance
=================
.. tabs::
.. group-tab:: Ubuntu / debian
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START searxng uwsgi-description ubuntu-20.04
:end-before: END searxng uwsgi-description ubuntu-20.04
@ -112,7 +161,7 @@ restart the uwsgi application.
.. group-tab:: Arch Linux
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START searxng uwsgi-description arch
:end-before: END searxng uwsgi-description arch
@ -120,16 +169,28 @@ restart the uwsgi application.
.. group-tab:: Fedora / RHEL
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START searxng uwsgi-description fedora
:end-before: END searxng uwsgi-description fedora
.. _uwsgi setup:
uWSGI setup
===========
Create the configuration ini-file according to your distribution and restart the
uwsgi application. As shown below, the :ref:`installation scripts` installs by
default:
- a uWSGI setup that listens on a socket and
- enables :ref:`cache busting <static_use_hash>`.
.. tabs::
.. group-tab:: Ubuntu / debian
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START searxng uwsgi-appini ubuntu-20.04
:end-before: END searxng uwsgi-appini ubuntu-20.04
@ -137,7 +198,7 @@ restart the uwsgi application.
.. group-tab:: Arch Linux
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START searxng uwsgi-appini arch
:end-before: END searxng uwsgi-appini arch
@ -145,6 +206,63 @@ restart the uwsgi application.
.. group-tab:: Fedora / RHEL
.. kernel-include:: $DOCS_BUILD/includes/searx.rst
.. kernel-include:: $DOCS_BUILD/includes/searxng.rst
:start-after: START searxng uwsgi-appini fedora
:end-before: END searxng uwsgi-appini fedora
.. _uWSGI Tyrant mode pitfalls:
Pitfalls of the Tyrant mode
===========================
The implementation of the process owners and groups in the `Tyrant mode`_ is
somewhat unusual and requires special consideration. In `Tyrant mode`_ mode the
Emperor will run the vassal using the UID/GID of the vassal configuration file
(user and group of the app ``.ini`` file).
.. _#2099@uWSGI: https://github.com/unbit/uwsgi/issues/2099
.. _#752@uWSGI: https://github.com/unbit/uwsgi/pull/752
.. _#2425uWSGI: https://github.com/unbit/uwsgi/issues/2425
Without option ``emperor-tyrant-initgroups=true`` in ``/etc/uwsgi.ini`` the
process won't get the additional groups, but this option is not available in
2.0.x branch (see `#2099@uWSGI`_) the feature `#752@uWSGI`_ has been merged (on
Oct. 2014) to the master branch of uWSGI but had never been released; the last
major release is from Dec. 2013, since the there had been only bugfix releases
(see `#2425uWSGI`_). To shorten up:
**In Tyrant mode, there is no way to get additional groups, and the uWSGI
process misses additional permissions that may be needed.**
For example on Fedora (RHEL): If you try to install a redis DB with socket
communication and you want to connect to it from the SearXNG uWSGI, you will see a
*Permission denied* in the log of your instance::
ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ...
ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied.
ERROR:searx.plugins.limiter: init limiter DB failed!!!
Even if your *searxng* user of the uWSGI process is added to additional groups
to give access to the socket from the redis DB::
$ groups searxng
searxng : searxng searxng-redis
To see the effective groups of the uwsgi process, you have to look at the status
of the process, by example::
$ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini'
searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini
searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini
Here you can see that the additional "Groups" of PID 186 are unset (missing gid
of ``searxng-redis``)::
$ cat /proc/186/task/186/status
...
Uid: 993 993 993 993
Gid: 993 993 993 993
FDSize: 128
Groups:
...

106
docs/admin/installation.rst
View File

@ -4,109 +4,19 @@
Installation
============
.. sidebar:: info
:ref:`installation switch2ng`
*You're spoilt for choice*, choose your preferred method of installation.
- :ref:`installation docker`
- :ref:`installation scripts`
- :ref:`installation basic`
The :ref:`installation basic` is good enough for intranet usage and it is a
excellent illustration of *how a SearXNG instance is build up*. If you place your
instance public to the internet you should really consider to install a
:ref:`filtron reverse proxy <filtron.sh>` and for privacy a :ref:`result proxy
<morty.sh>` is mandatory.
The :ref:`installation basic` is an excellent illustration of *how a SearXNG
instance is build up* (see :ref:`architecture uWSGI`). If you do not have any
special preferences, its recommend to use the :ref:`installation docker` or the
:ref:`installation scripts`.
Therefore, if you do not have any special preferences, its recommend to use the
:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling
box <toolboxing>` as described below.
.. attention::
.. _installation scripts:
Installation scripts
====================
.. sidebar:: Update OS first!
To avoid unwanted side effects, update your OS before installing SearXNG.
The following will install a setup as shown in :ref:`architecture`. First you
need to get a clone. The clone is only needed for the installation procedure
and some maintenance tasks (alternatively you can create your own fork).
For the installation procedure, use a *sudoer* login to run the scripts. If you
install from ``root``, take into account that the scripts are creating a
``searx``, a ``filtron`` and a ``morty`` user. In the installation procedure
these new created users do need read access to the clone of searx, which is not
the case if you clone into a folder below ``/root``.
.. code:: bash
$ cd ~/Downloads
$ git clone https://github.com/searxng/searxng.git searxng
$ cd searxng
.. sidebar:: further read
- :ref:`toolboxing`
- :ref:`update searxng`
- :ref:`inspect searxng`
**Install** :ref:`SearXNG service <searx.sh>`
This installs SearXNG as described in :ref:`installation basic`.
.. code:: bash
$ sudo -H ./utils/searx.sh install all
**Install** :ref:`filtron reverse proxy <filtron.sh>`
.. code:: bash
$ sudo -H ./utils/filtron.sh install all
**Install** :ref:`result proxy <morty.sh>`
.. code:: bash
$ sudo -H ./utils/morty.sh install all
If all services are running fine, you can add it to your HTTP server:
**Install** HTTP
- :ref:`installation apache`
- :ref:`installation nginx`
**Install** :ref:`external plugins <dev plugin>`
Use SearXNG's ``shell`` to install external plugins. In the example below we
install the SearXNG plugins from **The Green Web Foundation** `[ref]
<https://www.thegreenwebfoundation.org/news/searching-the-green-web-with-searx/>`__:
.. code:: bash
$ sudo -H ./utils/searx.sh shell
// exit with [CTRL-D]
(searx-pyenv) searx@ryzen:~$ pip install git+https://github.com/return42/tgwf-searx-plugins
In the :ref:`settings.yml` activate the ``plugins:`` section and add module
``only_show_green_results`` from tgwf-searx-plugins.
.. code:: yaml
plugins:
- only_show_green_results
.. _git stash: https://git-scm.com/docs/git-stash
.. tip::
About script's installation options have a look at chapter :ref:`toolboxing
setup`. How to brand your instance see chapter :ref:`settings global`. To
*stash* your instance's setup, `git stash`_ your clone's :origin:`.config.sh`
file .
SearXNG is growing rapidly, you should regularly read our :ref:`migrate and
stay tuned` section. If you want to upgrade an existing instance or migrate
from searx to SearXNG, you should read this section first!

40
docs/admin/morty.rst
View File

@ -1,40 +0,0 @@
.. _searxng morty:
=========================
How to setup result proxy
=========================
.. sidebar:: further reading
- :ref:`morty.sh`
.. _morty: https://github.com/asciimoo/morty
.. _morty's README: https://github.com/asciimoo/morty
By default SearXNG can only act as an image proxy for result images, but it is
possible to proxify all the result URLs with an external service, morty_.
To use this feature, morty has to be installed and activated in SearXNG's
``settings.yml``. Add the following snippet to your ``settings.yml`` and
restart searx:
.. code:: yaml
result_proxy:
url : http://127.0.0.1:3000/
key : !!binary "insert_your_morty_proxy_key_here"
Note that the example above (``http://127.0.0.1:3000``) is only for single-user
instances without a HTTP proxy. If your morty service is public, the url is the
address of the reverse proxy (e.g ``https://example.org/morty``).
For more information about *result proxy* have a look at *"SearXNG via filtron
plus morty"* in the :ref:`nginx <nginx searxng via filtron plus morty>` and
:ref:`apache <apache searxng via filtron plus morty>` sections.
``url``
Is the address of the running morty service.
``key``
Is an optional argument, see `morty's README`_ for more information.

142
docs/admin/update-searxng.rst
View File

@ -1,59 +1,115 @@
.. _update searxng:
=============
How to update
=============
How to update depends on the :ref:`installation` method. If you have used the
:ref:`installation scripts`, use ``update`` command from the scripts.
**Update** :ref:`SearXNG service <searx.sh>`
.. code:: sh
sudo -H ./utils/searx.sh update searx
**Update** :ref:`filtron reverse proxy <filtron.sh>`
.. code:: sh
sudo -H ./utils/filtron.sh update filtron
**Update** :ref:`result proxy <morty.sh>`
.. code:: bash
$ sudo -H ./utils/morty.sh update morty
.. _inspect searxng:
======================
How to inspect & debug
======================
===================
SearXNG maintenance
===================
.. sidebar:: further read
- :ref:`toolboxing`
- :ref:`Makefile`
- :ref:`uWSGI maintenance`
.. contents:: Contents
:depth: 2
:local:
:backlinks: entry
.. _update searxng:
How to update
=============
How to update depends on the :ref:`installation` method. If you have used the
:ref:`installation scripts`, use the ``update`` command from the :ref:`searxng.sh`
script.
.. code:: sh
sudo -H ./utils/searxng.sh instance update
.. _inspect searxng:
How to inspect & debug
======================
How to debug depends on the :ref:`installation` method. If you have used the
:ref:`installation scripts`, use ``inspect`` command from the scripts.
**Inspect** :ref:`SearXNG service <searx.sh>`
:ref:`installation scripts`, use the ``inspect`` command from the :ref:`searxng.sh`
script.
.. code:: sh
sudo -H ./utils/searx.sh inspect service
sudo -H ./utils/searxng.sh instance inspect
**Inspect** :ref:`filtron reverse proxy <filtron.sh>`
.. _migrate and stay tuned:
.. code:: sh
Migrate and stay tuned!
=======================
sudo -H ./utils/filtron.sh inspect service
.. sidebar:: info
**Inspect** :ref:`result proxy <morty.sh>`
- :pull:`1332`
- :pull:`456`
- :pull:`A comment about rolling release <446#issuecomment-954730358>`
.. code:: bash
SearXNG is a *rolling release*; each commit to the master branch is a release.
SearXNG is growing rapidly, the services and opportunities are change every now
and then, to name just a few:
$ sudo -H ./utils/morty.sh inspect service
- Bot protection has been switched from filtron to SearXNG's :ref:`limiter
<limiter>`, this requires a :ref:`Redis <settings redis>` database.
- The image proxy morty is no longer needed, it has been replaced by the
:ref:`image proxy <image_proxy>` from SearXNG.
- To save bandwith :ref:`cache busting <static_use_hash>` has been implemented.
To get in use, the ``static-expires`` needs to be set in the :ref:`uwsgi
setup`.
To stay tuned and get in use of the new features, instance maintainers have to
update the SearXNG code regularly (see :ref:`update searxng`). As the above
examples show, this is not always enough, sometimes services have to be set up
or reconfigured and sometimes services that are no longer needed should be
uninstalled.
.. hint::
First of all: SearXNG is installed by the script :ref:`searxng.sh`. If you
have old filtron, morty or searx setup you should consider complete
uninstall/reinstall.
remove obsolete services
------------------------
If your searx instance was installed *"Step by step"* or by the *"Installation
scripts"*, you need to undo the installation procedure completely. If you have
morty & filtron installed, it is recommended to uninstall these services also.
In case of scripts, to uninstall use the scripts from the origin you installed
searx from or try::
$ sudo -H ./utils/filtron.sh remove all
$ sudo -H ./utils/morty.sh remove all
$ sudo -H ./utils/searx.sh remove all
.. hint::
If you are migrate from searx take into account that the ``.config.sh`` is no
longer used.
Check after Installation
------------------------
Once you have done your installation, you can run a SearXNG *check* procedure,
to see if there are some left overs. In this example there exists a *old*
``/etc/searx/settings.yml``::
$ sudo -H ./utils/searxng.sh instance check
SearXNG checks
--------------
ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/
INFO: [OK] (old) account 'searx' does not exists
INFO: [OK] (old) account 'filtron' does not exists
INFO: [OK] (old) account 'morty' does not exists
...
INFO searx.shared : Use shared_simple implementation
INFO searx.shared.redis : connected redis DB --> default

40
docs/build-templates/searx.rst → docs/build-templates/searxng.rst
View File

@ -1,4 +1,4 @@
.. template evaluated by: ./utils/searx.sh docs
.. template evaluated by: ./utils/searxng.sh searxng.doc.rst
.. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$
.. START distro-packages
@ -65,7 +65,8 @@ ${fedora_build}
$ sudo -H useradd --shell /bin/bash --system \\
--home-dir \"$SERVICE_HOME\" \\
--comment 'Privacy-respecting metasearch engine' $SERVICE_USER
--comment 'Privacy-respecting metasearch engine' \\
$SERVICE_USER
$ sudo -H mkdir \"$SERVICE_HOME\"
$ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\"
@ -81,7 +82,8 @@ ${fedora_build}
.. code-block:: sh
$ sudo -H -u ${SERVICE_USER} -i
(${SERVICE_USER})$ git clone \"$GIT_URL\" \"$SEARX_SRC\"
(${SERVICE_USER})$ git clone \"$GIT_URL\" \\
\"$SEARXNG_SRC\"
.. END clone searxng
@ -93,8 +95,9 @@ ${fedora_build}
.. code-block:: sh
(${SERVICE_USER})$ python3 -m venv \"${SEARX_PYENV}\"
(${SERVICE_USER})$ echo \". ${SEARX_PYENV}/bin/activate\" >> \"$SERVICE_HOME/.profile\"
(${SERVICE_USER})$ python3 -m venv \"${SEARXNG_PYENV}\"
(${SERVICE_USER})$ echo \". ${SEARXNG_PYENV}/bin/activate\" \\
>> \"$SERVICE_HOME/.profile\"
.. END create virtualenv
@ -109,7 +112,7 @@ ${fedora_build}
$ sudo -H -u ${SERVICE_USER} -i
(${SERVICE_USER})$ command -v python && python --version
$SEARX_PYENV/bin/python
$SEARXNG_PYENV/bin/python
Python 3.8.1
# update pip's boilerplate ..
@ -119,7 +122,7 @@ ${fedora_build}
pip install -U pyyaml
# jump to SearXNG's working tree and install SearXNG into virtualenv
(${SERVICE_USER})$ cd \"$SEARX_SRC\"
(${SERVICE_USER})$ cd \"$SEARXNG_SRC\"
(${SERVICE_USER})$ pip install -e .
@ -134,24 +137,15 @@ ${fedora_build}
.. code-block:: sh
$ sudo -H mkdir -p \"$(dirname ${SEARXNG_SETTINGS_PATH})\"
$ sudo -H cp \"$SEARX_SRC/utils/templates/etc/searxng/settings.yml\" \\
$ sudo -H cp \"$SEARXNG_SRC/utils/templates/etc/searxng/settings.yml\" \\
\"${SEARXNG_SETTINGS_PATH}\"
.. group-tab:: searx/settings.yml
.. code-block:: sh
$ sudo -H mkdir -p \"$(dirname ${SEARXNG_SETTINGS_PATH})\"
$ sudo -H cp \"$SEARX_SRC/searx/settings.yml\" \\
\"${SEARXNG_SETTINGS_PATH}\"
.. tabs::
.. group-tab:: minimal setup
.. code-block:: sh
$ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \"$SEARXNG_SETTINGS_PATH\"
$ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \\
\"$SEARXNG_SETTINGS_PATH\"
.. END searxng config
@ -168,14 +162,14 @@ ${fedora_build}
# start webapp
$ sudo -H -u ${SERVICE_USER} -i
(${SERVICE_USER})$ cd ${SEARX_SRC}
(${SERVICE_USER})$ cd ${SEARXNG_SRC}
(${SERVICE_USER})$ export SEARXNG_SETTINGS_PATH=\"${SEARXNG_SETTINGS_PATH}\"
(${SERVICE_USER})$ python searx/webapp.py
# disable debug
$ sudo -H sed -i -e \"s/debug : True/debug : False/g\" \"$SEARXNG_SETTINGS_PATH\"
Open WEB browser and visit http://$SEARX_INTERNAL_HTTP . If you are inside a
Open WEB browser and visit http://$SEARXNG_INTERNAL_HTTP . If you are inside a
container or in a script, test with curl:
.. tabs::
@ -184,13 +178,13 @@ container or in a script, test with curl:
.. code-block:: sh
$ xdg-open http://$SEARX_INTERNAL_HTTP
$ xdg-open http://$SEARXNG_INTERNAL_HTTP
.. group-tab:: curl
.. code-block:: none
$ curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP
$ curl --location --verbose --head --insecure $SEARXNG_INTERNAL_HTTP
* Trying 127.0.0.1:8888...
* TCP_NODELAY set

2
docs/conf.py
View File

@ -195,5 +195,5 @@ html_show_sourcelink = True
# LaTeX ----------------------------------------------------------------
latex_documents = [
(master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")
(master_doc, "searxng-{}.tex".format(VERSION_STRING), html_title, author, "manual")
]

8
docs/dev/engine_overview.rst
View File

@ -66,11 +66,11 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin
.. table:: Common options in the engine setup (``settings.yml``)
:width: 100%
======================= =========== ===============================================
======================= =========== ==================================================
argument type information
======================= =========== ===============================================
======================= =========== ==================================================
name string name of search-engine
engine string name of searx-engine (filename without ``.py``)
engine string name of searxng-engine (file name without ``.py``)
enable_http bool enable HTTP (by default only HTTPS is enabled).
shortcut string shortcut of search-engine
timeout string specific timeout for search-engine
@ -78,7 +78,7 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin
proxies dict set proxies for a specific engine
(e.g. ``proxies : {http: socks5://proxy:port,
https: socks5://proxy:port}``)
======================= =========== ===============================================
======================= =========== ==================================================
.. _engine overrides:

189
docs/dev/lxcdev.rst
View File

@ -45,9 +45,7 @@ be set on a *production* system.
The scripts from :ref:`searx_utils` can divide in those to install and maintain
software:
- :ref:`searx.sh`
- :ref:`filtron.sh`
- :ref:`morty.sh`
- :ref:`searxng.sh`
and the script :ref:`lxc.sh`, with we can scale our installation, maintenance or
even development tasks over a stack of isolated containers / what we call the:
@ -73,7 +71,7 @@ once:
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ snap install lxd
$ lxd init --auto
@ -85,28 +83,28 @@ fork:
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ cd ~/Downloads
$ git clone https://github.com/searxng/searxng.git searxng
$ cd searxng
The :ref:`lxc-searx.env` consists of several images, see ``export
LXC_SUITE=(...`` near by :origin:`utils/lxc-searx.env#L19`. For this blog post
The :ref:`lxc-searxng.env` consists of several images, see ``export
LXC_SUITE=(...`` near by :origin:`utils/lxc-searxng.env#L19`. For this blog post
we exercise on a archlinux_ image. The container of this image is named
``searx-archlinux``. Lets build the container, but be sure that this container
``searxng-archlinux``. Lets build the container, but be sure that this container
does not already exists, so first lets remove possible old one:
.. tabs::
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh remove searx-archlinux
$ sudo -H ./utils/lxc.sh build searx-archlinux
$ sudo -H ./utils/lxc.sh remove searxng-archlinux
$ sudo -H ./utils/lxc.sh build searxng-archlinux
.. sidebar:: The ``searx-archlinux`` container
.. sidebar:: The ``searxng-archlinux`` container
is the base of all our exercises here.
@ -117,9 +115,9 @@ In this container we install all services :ref:`including searx, morty & filtron
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh install suite searx-archlinux
$ sudo -H ./utils/lxc.sh install suite searxng-archlinux
To proxy HTTP from filtron and morty in the container to the outside of the
container, install nginx into the container. Once for the bot blocker filtron:
@ -128,9 +126,9 @@ container, install nginx into the container. Once for the bot blocker filtron:
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
./utils/filtron.sh nginx install
...
INFO: got 429 from http://10.174.184.156/searx
@ -141,9 +139,9 @@ and once for the content sanitizer (content proxy morty):
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
./utils/morty.sh nginx install
...
INFO: got 200 from http://10.174.184.156/morty/
@ -154,7 +152,7 @@ and once for the content sanitizer (content proxy morty):
blocker (filtron) and WEB content sanitizer (content proxy morty), both are
needed for a *privacy protecting* search engine.
On your system, the IP of your ``searx-archlinux`` container differs from
On your system, the IP of your ``searxng-archlinux`` container differs from
http://10.174.184.156/searx, just open the URL reported in your installation
protocol in your WEB browser from the desktop to test the instance from outside
of the container.
@ -169,27 +167,27 @@ In containers, work as usual
Usually you open a root-bash using ``sudo -H bash``. In case of LXC containers
open the root-bash in the container using ``./utils/lxc.sh cmd
searx-archlinux``:
searxng-archlinux``:
.. tabs::
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux bash
INFO: [searx-archlinux] bash
[root@searx-archlinux searx]# pwd
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash
INFO: [searxng-archlinux] bash
[root@searxng-archlinux searx]# pwd
/share/searxng
The prompt ``[root@searx-archlinux ...]`` signals, that you are the root user in
the searx-container. To debug the running SearXNG instance use:
The prompt ``[root@searxng-archlinux ...]`` signals, that you are the root user in
the searxng-container. To debug the running SearXNG instance use:
.. tabs::
.. group-tab:: root@searx-archlinux
.. group-tab:: root@searxng-archlinux
.. code:: sh
.. code:: bash
$ ./utils/searx.sh inspect service
...
@ -202,56 +200,42 @@ above. You can stop monitoring using ``CTRL-C``, this also disables the *"debug
option"* in SearXNG's settings file and restarts the SearXNG uwsgi application.
To debug services from filtron and morty analogous use:
.. tabs::
.. group-tab:: root@searx-archlinux
.. code:: sh
$ ./utils/filtron.sh inspect service
$ ./utils/morty.sh inspect service
Another point we have to notice is that each service (:ref:`SearXNG <searx.sh>`,
:ref:`filtron <filtron.sh>` and :ref:`morty <morty.sh>`) runs under dedicated
system user account with the same name (compare :ref:`create searxng user`). To
get a shell from theses accounts, simply call one of the scripts:
Another point we have to notice is that the service (:ref:`SearXNG <searxng.sh>`
runs under dedicated system user account with the same name (compare
:ref:`create searxng user`). To get a shell from theses accounts, simply call:
.. tabs::
.. group-tab:: root@searx-archlinux
.. group-tab:: root@searxng-archlinux
.. code:: sh
.. code:: bash
$ ./utils/searx.sh shell
$ ./utils/filtron.sh shell
$ ./utils/morty.sh shell
$ ./utils/searxng.sh instance cmd bash
To get in touch, open a shell from the service user (searx@searx-archlinux):
To get in touch, open a shell from the service user (searxng@searxng-archlinux):
.. tabs::
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
./utils/searx.sh shell
// exit with [CTRL-D]
(searx-pyenv) [searx@searx-archlinux ~]$ ...
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh instance cmd bash
INFO: [searxng-archlinux] ./utils/searxng.sh instance cmd bash
[searxng@searxng-archlinux ~]$
The prompt ``[searx@searx-archlinux]`` signals that you are logged in as system
user ``searx`` in the ``searx-archlinux`` container and the python *virtualenv*
``(searx-pyenv)`` environment is activated.
The prompt ``[searxng@searxng-archlinux]`` signals that you are logged in as system
user ``searx`` in the ``searxng-archlinux`` container and the python *virtualenv*
``(searxng-pyenv)`` environment is activated.
.. tabs::
.. group-tab:: searx@searx-archlinux
.. group-tab:: searxng@searxng-archlinux
.. code:: sh
(searx-pyenv) [searx@searx-archlinux ~]$ pwd
/usr/local/searx
.. code:: bash
(searxng-pyenv) [searxng@searxng-archlinux ~]$ pwd
/usr/local/searxng
Wrap production into developer suite
@ -262,23 +246,22 @@ from a LXC container (which is quite ready for production) into a developer
suite. For this, we have to keep an eye on the :ref:`installation basic`:
- SearXNG setup in: ``/etc/searxng/settings.yml``
- SearXNG user's home: ``/usr/local/searx``
- virtualenv in: ``/usr/local/searx/searx-pyenv``
- SearXNG software in: ``/usr/local/searx/searx-src``
- SearXNG user's home: ``/usr/local/searxng``
- virtualenv in: ``/usr/local/searxng/searxng-pyenv``
- SearXNG software in: ``/usr/local/searxng/searxng-src``
With the use of the :ref:`searx.sh` the SearXNG service was installed as
With the use of the :ref:`searxng.sh` the SearXNG service was installed as
:ref:`uWSGI application <searxng uwsgi>`. To maintain this service, we can use
``systemctl`` (compare :ref:`service architectures on distributions <uwsgi
configuration>`).
``systemctl`` (compare :ref:`uWSGI maintenance`).
.. tabs::
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
systemctl stop uwsgi@searx
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
systemctl stop uwsgi@searxng
With the command above, we stopped the SearXNG uWSGI-App in the archlinux
container.
@ -291,29 +274,29 @@ least you should attend the settings of ``uid``, ``chdir``, ``env`` and
env = SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml
http = 127.0.0.1:8888
chdir = /usr/local/searx/searx-src/searx
virtualenv = /usr/local/searx/searx-pyenv
pythonpath = /usr/local/searx/searx-src
chdir = /usr/local/searxng/searxng-src/searx
virtualenv = /usr/local/searxng/searxng-pyenv
pythonpath = /usr/local/searxng/searxng-src
If you have read the :ref:`"Good to know section" <lxc.sh>` you remember, that
each container shares the root folder of the repository and the command
``utils/lxc.sh cmd`` handles relative path names **transparent**. To wrap the
SearXNG installation into a developer one, we simple have to create a smylink to
the **transparent** reposetory from the desktop. Now lets replace the
repository at ``searx-src`` in the container with the working tree from outside
repository at ``searxng-src`` in the container with the working tree from outside
of the container:
.. tabs::
.. group-tab:: container becomes a developer suite
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
ln -s /share/searx/ /usr/local/searx/searx-src
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
ln -s /share/searx/ /usr/local/searxng/searxng-src
Now we can develop as usual in the working tree of our desktop system. Every
time the software was changed, you have to restart the SearXNG service (in the
@ -323,9 +306,9 @@ conatiner):
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
systemctl restart uwsgi@searx
@ -338,30 +321,30 @@ daily usage:
To *inspect* the SearXNG instance (already described above):
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
./utils/searx.sh inspect service
Run :ref:`makefile`, e.g. to test inside the container:
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
make test
To install all prerequisites needed for a :ref:`buildhosts`:
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
./utils/searx.sh install buildhost
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
./utils/searxng.sh install buildhost
To build the docs on a buildhost :ref:`buildhosts`:
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh cmd searx-archlinux \
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
make docs.html
.. _lxcdev summary:
@ -371,18 +354,18 @@ Summary
We build up a fully functional SearXNG suite in a archlinux container:
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh install suite searx-archlinux
$ sudo -H ./utils/lxc.sh install suite searxng-archlinux
To access HTTP from the desktop we installed nginx for the services inside the
conatiner:
.. tabs::
.. group-tab:: [root@searx-archlinux]
.. group-tab:: [root@searxng-archlinux]
.. code:: sh
.. code:: bash
$ ./utils/filtron.sh nginx install
$ ./utils/morty.sh nginx install
@ -393,12 +376,12 @@ the container :
.. tabs::
.. group-tab:: [root@searx-archlinux]
.. group-tab:: [root@searxng-archlinux]
.. code:: sh
.. code:: bash
$ mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old
$ ln -s /share/searx/ /usr/local/searx/searx-src
$ mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old
$ ln -s /share/searx/ /usr/local/searxng/searxng-src
$ systemctl restart uwsgi@searx
To get information about the searxNG suite in the archlinux container we can
@ -408,13 +391,13 @@ use:
.. group-tab:: desktop
.. code:: sh
.. code:: bash
$ sudo -H ./utils/lxc.sh show suite searx-archlinux
$ sudo -H ./utils/lxc.sh show suite searxng-archlinux
...
[searx-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx
[searx-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/
[searx-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/
[searx-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65]
[searxng-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx
[searxng-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/
[searxng-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/
[searxng-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65]
...

26
docs/dev/makefile.rst
View File

@ -40,7 +40,7 @@ We do no longer need to build up the virtualenv manually. Jump into your git
working tree and release a ``make install`` to get a virtualenv with a
*developer install* of SearXNG (:origin:`setup.py`). ::
$ cd ~/searx-clone
$ cd ~/searxng-clone
$ make install
PYENV [virtualenv] installing ./requirements*.txt into local/py3
...
@ -288,27 +288,3 @@ To filter out HTTP redirects (3xx_)::
https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0
https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None
--
``make pybuild``
================
.. _PyPi: https://pypi.org/
.. _twine: https://twine.readthedocs.io/en/latest/
Build Python packages in ``./dist/py``::
$ make pybuild
...
BUILD pybuild
running sdist
running egg_info
...
running bdist_wheel
$ ls ./dist
searx-0.18.0-py3-none-any.whl searx-0.18.0.tar.gz
To upload packages to PyPi_, there is also a ``pypi.upload`` target (to test use
``pypi.upload.test``). Since you are not the owner of :pypi:`searx` you will
never need to upload.

6
docs/dev/offline_engines.rst
View File

@ -55,10 +55,10 @@ admins can install packages in advance.
If there is a need to install additional packages in *Python's Virtual
Environment* of your SearXNG instance you need to switch into the environment
(:ref:`searx-src`) first, for this you can use :ref:`searx.sh`::
(:ref:`searxng-src`) first, for this you can use :ref:`searxng.sh`::
$ sudo utils/searx.sh shell
(searx-pyenv)$ pip install ...
$ sudo utils/searxng.sh instance cmd bash
(searxng-pyenv)$ pip install ...
Private engines (Security)

27
docs/dev/plugins.rst
View File

@ -33,17 +33,26 @@ Example plugin
External plugins
================
External plugins are standard python modules implementing all the requirements of the standard plugins.
Plugins can be enabled by adding them to :ref:`settings.yml`'s ``plugins`` section.
Example external plugin can be found `here <https://github.com/asciimoo/searx_external_plugin_example>`_.
SearXNG supports *external plugins* / there is no need to install one, SearXNG
runs out of the box. But to demonstrate; in the example below we install the
SearXNG plugins from *The Green Web Foundation* `[ref]
<https://www.thegreenwebfoundation.org/news/searching-the-green-web-with-searx/>`__:
Register your plugin
====================
.. code:: bash
$ sudo utils/searxng.sh instance cmd bash
(searxng-pyenv)$ pip install git+https://github.com/return42/tgwf-searx-plugins
In the :ref:`settings.yml` activate the ``plugins:`` section and add module
``only_show_green_results`` from ``tgwf-searx-plugins``.
.. code:: yaml
plugins:
...
- only_show_green_results
...
To enable your plugin register your plugin in
searx > plugin > __init__.py.
And at the bottom of the file add your plugin like.
``plugins.register(name_of_python_file)``
Plugin entry points
===================

14
docs/dev/quickstart.rst
View File

@ -10,7 +10,7 @@ Development Quickstart
SearXNG loves developers, just clone and start hacking. All the rest is done for
you simply by using :ref:`make <makefile>`.
.. code:: sh
.. code:: bash
git clone https://github.com/searxng/searxng.git searxng
@ -27,21 +27,21 @@ to our ":ref:`how to contribute`" guideline.
If you implement themes, you will need to setup a :ref:`make node.env` once:
.. code:: sh
.. code:: bash
make node.env
Before you call *make run* (2.), you need to compile the modified styles and
JavaScript:
.. code:: sh
.. code:: bash
make themes.all
Alternatively you can also compile selective the theme you have modified,
e.g. the *simple* theme.
.. code:: sh
.. code:: bash
make themes.simple
@ -52,7 +52,7 @@ e.g. the *simple* theme.
If you finished your *tests* you can start to commit your changes. To separate
the modified source code from the build products first run:
.. code:: sh
.. code:: bash
make static.build.restore
@ -60,13 +60,13 @@ This will restore the old build products and only your changes of the code
remain in the working tree which can now be added & commited. When all sources
are commited, you can commit the build products simply by:
.. code:: sh
.. code:: bash
make static.build.commit
Commiting the build products should be the last step, just before you send us
your PR. There is also a make target to rewind this last build commit:
.. code:: sh
.. code:: bash
make static.build.drop

80
docs/utils/filtron.sh.rst
View File

@ -1,80 +0,0 @@
.. _filtron.sh:
====================
``utils/filtron.sh``
====================
.. sidebar:: further reading
- :ref:`searxng filtron`
- :ref:`architecture`
- :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
<installation apache>`)
.. _Go: https://golang.org/
.. _filtron: https://github.com/searxng/filtron
.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md
To simplify installation and maintenance of a filtron instance you can use the
script :origin:`utils/filtron.sh`. In most cases you will install filtron_
simply by running the command:
.. code:: bash
sudo -H ./utils/filtron.sh install all
The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_
into this user account:
#. Create a separated user account (``filtron``).
#. Download and install Go_ binary in user's $HOME (``~filtron``).
#. Install filtron with the package management from Go_ (``go get -v -u
github.com/searxng/filtron``)
#. Setup a proper rule configuration :origin:`[ref]
<utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``).
#. Setup a systemd service unit :origin:`[ref]
<utils/templates/lib/systemd/system/filtron.service>`
(``/lib/systemd/system/filtron.service``).
Create user
===========
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START create user
:end-before: END create user
Install go
==========
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START install go
:end-before: END install go
Install filtron
===============
Install :origin:`rules.json <utils/templates/etc/filtron/rules.json>` at
``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and
install filtron software and systemd unit:
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START install filtron
:end-before: END install filtron
.. kernel-include:: $DOCS_BUILD/includes/filtron.rst
:start-after: START install systemd unit
:end-before: END install systemd unit
.. _filtron.sh overview:
Overview
========
The ``--help`` output of the script is largely self-explanatory
(:ref:`toolboxing common`):
.. program-output:: ../utils/filtron.sh --help

48
docs/utils/index.rst
View File

@ -1,52 +1,30 @@
.. _searx_utils:
.. _toolboxing:
===================
Admin's tooling box
===================
==================
DevOps tooling box
==================
In the folder :origin:`utils/` we maintain some tools useful for administrators.
In the folder :origin:`utils/` we maintain some tools useful for administrators
and developers.
.. toctree::
:maxdepth: 2
:caption: Contents
searx.sh
filtron.sh
morty.sh
searxng.sh
lxc.sh
.. _toolboxing common:
Common command environments
===========================
Common commands & environment
=============================
Scripts to maintain services often dispose of common commands and environments.
``shell`` : command
Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for
troubleshooting.
``inspect service`` : command
Shows status and log of the service, most often you have a option to enable
more verbose debug logs. Very helpful for debugging, but be careful not to
enable debugging in a production environment!
The scripts in our tooling box often dispose of common environments:
``FORCE_TIMEOUT`` : environment
Sets timeout for interactive prompts. If you want to run a script in batch
job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a
reverse proxy for filtron on all containers of the :ref:`SearXNG suite
<lxc-searx.env>` use ::
SearXNG server and nginx proxy on all containers of the :ref:`SearXNG suite
<lxc-searxng.env>` use::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install
.. _toolboxing setup:
Tooling box setup
=================
The main setup is done in the :origin:`.config.sh` (read also :ref:`settings
global`).
.. literalinclude:: ../../.config.sh
:language: bash
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install all
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx

52
docs/utils/lxc.sh.rst
View File

@ -23,7 +23,7 @@
With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of
containers, what we call the: *lxc suite*. The *SearXNG suite*
(:origin:`lxc-searx.env <utils/lxc-searx.env>`) is loaded by default, every time
(:origin:`lxc-searxng.env <utils/lxc-searxng.env>`) is loaded by default, every time
you start the ``lxc.sh`` script (*you do not need to care about*).
Before you can start with containers, you need to install and initiate LXD_
@ -49,7 +49,7 @@ help>`.
If you do not want to build all containers, **you can build just one**::
$ sudo -H ./utils/lxc.sh build searx-ubu1804
$ sudo -H ./utils/lxc.sh build searxng-archlinux
*Good to know ...*
@ -62,9 +62,9 @@ of::
In the containers, you can run what ever you want, e.g. to start a bash use::
$ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash
INFO: [searx-ubu1804] bash
root@searx-ubu1804:/share/searx#
$ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash
INFO: [searxng-archlinux] bash
[root@searxng-archlinux SearXNG]#
If there comes the time you want to **get rid off all** the containers and
**clean up local images** just type::
@ -121,28 +121,26 @@ Install suite
=============
To install the complete :ref:`SearXNG suite (includes searx, morty & filtron)
<lxc-searx.env>` into all LXC_ use::
<lxc-searxng.env>` into all LXC_ use::
$ sudo -H ./utils/lxc.sh install suite
The command above installs a SearXNG suite (see :ref:`installation scripts`). To
get the IP (URL) of the filtron service in the containers use ``show suite``
The command above installs a SearXNG suite (see :ref:`installation scripts`).
To :ref:`install a nginx <installation nginx>` reverse proxy (or alternatively
use :ref:`apache <installation apache>`)::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
To get the IP (URL) of the SearXNG service in the containers use ``show suite``
command. To test instances from containers just open the URLs in your
WEB-Browser::
$ sudo ./utils/lxc.sh show suite | grep filtron
[searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx
[searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx
[searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx
[searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx
[searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx
[searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx
$ sudo ./utils/lxc.sh show suite | grep SEARXNG_URL
To :ref:`install a nginx <installation nginx>` reverse proxy for filtron and
morty use (or alternatively use :ref:`apache <installation apache>`)::
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install
sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install
[searxng-ubu2110] SEARXNG_URL : http://n.n.n.147/searxng
[searxng-ubu2004] SEARXNG_URL : http://n.n.n.246/searxng
[searxnggfedora35] SEARXNG_URL : http://n.n.n.140/searxng
[searxng-archlinux] SEARXNG_URL : http://n.n.n.165/searxng
Running commands
@ -152,8 +150,8 @@ Running commands
:ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the
Makefile target ``test`` in the archlinux_ container::
sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost
sudo -H ./utils/lxc.sh cmd searx-archlinux make test
sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh install buildhost
sudo -H ./utils/lxc.sh cmd searxng-archlinux make test
Setup SearXNG buildhost
@ -164,11 +162,11 @@ The installation procedure to set up a :ref:`build host<buildhosts>` takes its
time. Installation in all containers will take more time (time for another cup
of coffee).::
sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost
sudo -H ./utils/lxc.sh cmd -- ./utils/searxng.sh install buildhost
To build (live) documentation inside a archlinux_ container::
sudo -H ./utils/lxc.sh cmd searx-archlinux make docs.clean docs.live
sudo -H ./utils/lxc.sh cmd searxng-archlinux make docs.clean docs.live
...
[I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
@ -176,7 +174,7 @@ To get IP of the container and the port number *live docs* is listening::
$ sudo ./utils/lxc.sh show suite | grep docs.live
...
[searx-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/
[searxng-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/
.. _lxc.sh help:
@ -189,10 +187,10 @@ The ``--help`` output of the script is largely self-explanatory:
.. program-output:: ../utils/lxc.sh --help
.. _lxc-searx.env:
.. _lxc-searxng.env:
SearXNG suite
=============
.. literalinclude:: ../../utils/lxc-searx.env
.. literalinclude:: ../../utils/lxc-searxng.env
:language: bash

80
docs/utils/morty.sh.rst
View File

@ -1,80 +0,0 @@
.. _morty: https://github.com/asciimoo/morty
.. _morty's README: https://github.com/asciimoo/morty
.. _Go: https://golang.org/
.. _morty.sh:
==================
``utils/morty.sh``
==================
.. sidebar:: further reading
- :ref:`architecture`
- :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache
<installation apache>`)
- :ref:`searxng morty`
To simplify installation and maintenance of a morty_ instance you can use the
script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by
running the command:
.. code:: bash
sudo -H ./utils/morty.sh install all
The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_
into this user account:
#. Create a separated user account (``morty``).
#. Download and install Go_ binary in user's $HOME (``~morty``).
#. Install morty_ with the package management from Go_ (``go get -v -u
github.com/asciimoo/morty``)
#. Setup a systemd service unit :origin:`[ref]
<utils/templates/lib/systemd/system/morty.service>`
(``/lib/systemd/system/morty.service``).
.. hint::
To add morty to your SearXNG instance read chapter :ref:`searxng morty`.
Create user
===========
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START create user
:end-before: END create user
Install go
==========
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START install go
:end-before: END install go
Install morty
=============
Install morty software and systemd unit:
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START install morty
:end-before: END install morty
.. kernel-include:: $DOCS_BUILD/includes/morty.rst
:start-after: START install systemd unit
:end-before: END install systemd unit
.. _morty.sh overview:
Overview
========
The ``--help`` output of the script is largely self-explanatory
(:ref:`toolboxing common`):
.. program-output:: ../utils/morty.sh --help

39
docs/utils/searx.sh.rst
View File

@ -1,39 +0,0 @@
.. _searx.sh:
==================
``utils/searx.sh``
==================
.. sidebar:: further reading
- :ref:`architecture`
- :ref:`installation`
- :ref:`installation nginx`
- :ref:`installation apache`
To simplify installation and maintenance of a SearXNG instance you can use the
script :origin:`utils/searx.sh`.
Install
=======
In most cases you will install SearXNG simply by running the command:
.. code:: bash
sudo -H ./utils/searx.sh install all
The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs SearXNG
into this user account. The installation is described in chapter
:ref:`installation basic`.
.. _intranet reverse proxy:
Overview
========
The ``--help`` output of the script is largely self-explanatory
(:ref:`toolboxing common`):
.. program-output:: ../utils/searx.sh --help

36
docs/utils/searxng.sh.rst Normal file
View File

@ -0,0 +1,36 @@
.. _searxng.sh:
====================
``utils/searxng.sh``
====================
.. sidebar:: further reading
- :ref:`architecture`
- :ref:`installation`
- :ref:`installation nginx`
- :ref:`installation apache`
To simplify the installation and maintenance of a SearXNG instance you can use the
script :origin:`utils/searxng.sh`.
Install
=======
In most cases you will install SearXNG simply by running the command:
.. code:: bash
sudo -H ./utils/searx.sh install all
The installation is described in chapter :ref:`installation basic`.
.. _searxng.sh overview:
Overview
========
The ``--help`` output of the script is largely self-explanatory:
.. program-output:: ../utils/searxng.sh --help

4
manage
View File

@ -416,9 +416,7 @@ docs.prebuild() {
set -e
[ "$VERBOSE" = "1" ] && set -x
mkdir -p "${DOCS_BUILD}/includes"
./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
pyenv.cmd searxng_extra/docs_prebuild
)
dump_return $?

6
searx/shared/redisdb.py
View File

@ -19,10 +19,13 @@ A redis DB connect can be tested by::
"""
import os
import pwd
import logging
import redis
from searx import get_setting
logger = logging.getLogger('searx.shared.redis')
_client = None
@ -42,6 +45,7 @@ def init():
logger.info("connected redis DB --> %s", c.acl_whoami())
return True
except redis.exceptions.ConnectionError as exc:
logger.error("can't connet redis DB ...")
_pw = pwd.getpwuid(os.getuid())
logger.error("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid)
logger.error(" %s", exc)
return False

527
utils/filtron.sh
View File

@ -4,56 +4,19 @@
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/lib_go.sh
source "${REPO_ROOT}/utils/lib_go.sh"
# shellcheck source=utils/lib_install.sh
source "${REPO_ROOT}/utils/lib_install.sh"
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
FILTRON_ETC="/etc/filtron"
FILTRON_RULES="$FILTRON_ETC/rules.json"
FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/filtron/rules.json}"
FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"
# The filtron target is the SearXNG installation, listenning on server.port at
# server.bind_address. The default of FILTRON_TARGET is taken from the YAML
# configuration, do not change this value without reinstalling the entire
# SearXNG suite including filtron & morty.
FILTRON_TARGET="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}"
SERVICE_NAME="filtron"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
GO_ENV="${SERVICE_HOME}/.go_env"
GO_VERSION="go1.17.2"
APACHE_FILTRON_SITE="searxng.conf"
NGINX_FILTRON_SITE="searxng.conf"
# shellcheck disable=SC2034
CONFIG_FILES=(
"${FILTRON_RULES}"
"${SERVICE_SYSTEMD_UNIT}"
)
APACHE_FILTRON_SITE="searx.conf"
NGINX_FILTRON_SITE="searx.conf"
# ----------------------------------------------------------------------------
usage() {
@ -62,248 +25,45 @@ usage() {
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") shell
$(basename "$0") install [all|user|rules]
$(basename "$0") reinstall all
$(basename "$0") update [filtron]
$(basename "$0") remove [all]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-on|debug-off]
$(basename "$0") apache [install|remove]
$(basename "$0") nginx [install|remove]
$(basename "$0") remove all]
$(basename "$0") apache remove
$(basename "$0") nginx remove
shell
start interactive shell from user ${SERVICE_USER}
install / remove
:all: complete setup of filtron service
:user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
:rules: reinstall filtron rules $FILTRON_RULES
install
:check: check the filtron installation
reinstall:
:all: runs 'install/remove all'
update filtron
Update filtron installation ($SERVICE_HOME)
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
show service status and log
option
set one of the available options
apache (${PUBLIC_URL})
:install: apache site with a reverse proxy (ProxyPass)
:remove: apache site ${APACHE_FILTRON_SITE}
nginx (${PUBLIC_URL})
:install: nginx site with a reverse proxy (ProxyPass)
:remove: nginx site ${NGINX_FILTRON_SITE}
filtron rules: ${FILTRON_RULES_TEMPLATE}
---- sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
FILTRON_TARGET : ${FILTRON_TARGET}
FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN}
FILTRON_URL_PATH : ${FILTRON_URL_PATH}
remove all : drop all components of the filtron service
apache remove : drop apache site ${APACHE_FILTRON_SITE}
nginx remove : drop nginx site ${NGINX_FILTRON_SITE}
EOF
install_log_searx_instance
[[ -n ${1} ]] && err_msg "$1"
}
main() {
required_commands \
sudo install git wget curl \
|| exit
local _usage="unknown or missing $1 command $2"
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
shell)
sudo_or_exit
interactive_shell "${SERVICE_USER}"
;;
inspect)
case $2 in
service)
sudo_or_exit
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
reinstall)
rst_title "re-install $SERVICE_NAME" part
sudo_or_exit
case $2 in
all)
remove_all
install_all
;;
*) usage "$_usage"; exit 42;;
esac ;;
install)
rst_title "$SERVICE_NAME" part
sudo_or_exit
case $2 in
check)
rst_title "Check filtron installation" part
install_check
;;
all) install_all ;;
user) assert_user ;;
rules)
install_rules
systemd_restart_service "${SERVICE_NAME}"
;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
sudo_or_exit
case $2 in
filtron) update_filtron ;;
*) usage "$_usage"; exit 42;;
esac ;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
user) drop_service_account "${SERVICE_USER}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
activate)
sudo_or_exit
case $2 in
service) systemd_activate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) systemd_deactivate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
nginx)
sudo_or_exit
case $2 in
install) install_nginx_site ;;
remove) remove_nginx_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
sudo_or_exit
case $2 in
debug-on) echo; enable_debug ;;
debug-off) echo; disable_debug ;;
*) usage "$_usage"; exit 42;;
esac ;;
doc) rst-doc ;;
*) usage "unknown or missing command $1"; exit 42;;
esac
}
install_all() {
rst_title "Install $SERVICE_NAME (service)"
assert_user
wait_key
go.golang "${GO_VERSION}" "${SERVICE_USER}"
wait_key
install_filtron
install_rules
wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
wait_key
echo
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}"
fi
if apache_is_installed; then
info_msg "Apache is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_apache_site
fi
elif nginx_is_installed; then
info_msg "nginx is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_nginx_site
fi
fi
if ask_yn "Do you want to inspect the installation?" Ny; then
inspect_service
fi
}
install_check() {
if service_account_is_available "$SERVICE_USER"; then
info_msg "service account $SERVICE_USER available."
else
err_msg "service account $SERVICE_USER not available!"
fi
if go_is_available "$SERVICE_USER"; then
info_msg "~$SERVICE_USER: go is installed"
else
err_msg "~$SERVICE_USER: go is not installed"
fi
if filtron_is_installed; then
info_msg "~$SERVICE_USER: filtron app is installed"
else
err_msg "~$SERVICE_USER: filtron app is not installed!"
fi
if ! service_is_available "http://${FILTRON_API}"; then
err_msg "API not available at: http://${FILTRON_API}"
fi
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}"
fi
if service_is_available "http://${FILTRON_TARGET}" ; then
info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
fi
if ! service_is_available "${PUBLIC_URL}"; then
warn_msg "Public service at ${PUBLIC_URL} is not available!"
if ! in_container; then
warn_msg "Check if public name is correct and routed or use the public IP from above."
fi
fi
if [[ "${GO_VERSION}" > "$(go_version)" ]]; then
warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least"
warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all"
else
info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)"
fi
if [ -f "${APACHE_SITES_AVAILABLE}/searx.conf" ]; then
warn_msg "old searx.conf apache site exists"
fi
if [ -f "${NGINX_APPS_AVAILABLE}/searx.conf" ]; then
warn_msg "old searx.conf nginx site exists"
fi
}
go_version(){
go.version "${SERVICE_USER}"
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
@ -321,219 +81,6 @@ installations that were installed with this script."
fi
}
assert_user() {
rst_title "user $SERVICE_USER" section
echo
tee_stderr 1 <<EOF | bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "$SERVICE_HOME" \
--comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER
mkdir "$SERVICE_HOME"
chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
groups $SERVICE_USER
EOF
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
export SERVICE_HOME
echo "export SERVICE_HOME=$SERVICE_HOME"
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
touch "$GO_ENV"
grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile
EOF
}
filtron_is_installed() {
[[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
}
install_filtron() {
rst_title "Install filtron in user's ~/go-apps" section
echo
go.install github.com/searxng/filtron@latest "${SERVICE_USER}"
}
update_filtron() {
rst_title "Update filtron" section
echo
go.install github.com/searxng/filtron@latest "${SERVICE_USER}"
}
install_rules() {
rst_title "Install filtron rules"
echo
if [[ ! -f "${FILTRON_RULES}" ]]; then
info_msg "install rules ${FILTRON_RULES_TEMPLATE}"
info_msg " --> ${FILTRON_RULES}"
mkdir -p "$(dirname "${FILTRON_RULES}")"
cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
return
fi
if cmp --silent "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"; then
info_msg "${FILTRON_RULES} is up to date with"
info_msg "${FILTRON_RULES_TEMPLATE}"
return
fi
rst_para "Diff between origin's rules file (+) and current (-):"
echo "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"
$DIFF_CMD "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"
local action
choose_one action "What should happen to the rules file? " \
"keep configuration unchanged" \
"use origin rules" \
"start interactive shell"
case $action in
"keep configuration unchanged")
info_msg "leave rules file unchanged"
;;
"use origin rules")
backup_file "${FILTRON_RULES}"
info_msg "install origin rules"
cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
;;
"start interactive shell")
backup_file "${FILTRON_RULES}"
echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
sudo -H -i
rst_para 'Diff between new rules file (-) and current (+):'
echo
$DIFF_CMD "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
wait_key
;;
esac
}
inspect_service() {
rst_title "service status & log"
cat <<EOF
sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
FILTRON_TARGET : ${FILTRON_TARGET}
FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN}
FILTRON_URL_PATH : ${FILTRON_URL_PATH}
EOF
install_log_searx_instance
install_check
if in_container; then
lxc_suite_info
else
info_msg "public URL --> ${PUBLIC_URL}"
info_msg "internal URL --> http://${FILTRON_LISTEN}"
fi
local _debug_on
if ask_yn "Enable filtron debug mode?"; then
enable_debug
_debug_on=1
fi
echo
systemctl --no-pager -l status "${SERVICE_NAME}"
echo
info_msg "public URL --> ${PUBLIC_URL}"
# shellcheck disable=SC2059
printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
read -r -s -n1 -t 5
echo
while true; do
trap break 2
journalctl -f -u "${SERVICE_NAME}"
done
if [[ $_debug_on == 1 ]]; then
disable_debug
fi
return 0
}
enable_debug() {
info_msg "try to enable debug mode ..."
python <<EOF
import sys, json
debug = {
u'name': u'debug request'
, u'filters': []
, u'interval': 0
, u'limit': 0
, u'actions': [{u'name': u'log'}]
}
with open('$FILTRON_RULES') as rules:
j = json.load(rules)
pos = None
for i in range(len(j)):
if j[i].get('name') == 'debug request':
pos = i
break
if pos is not None:
j[pos] = debug
else:
j.append(debug)
with open('$FILTRON_RULES', 'w') as rules:
json.dump(j, rules, indent=2, sort_keys=True)
EOF
systemctl restart "${SERVICE_NAME}.service"
}
disable_debug() {
info_msg "try to disable debug mode ..."
python <<EOF
import sys, json
with open('$FILTRON_RULES') as rules:
j = json.load(rules)
pos = None
for i in range(len(j)):
if j[i].get('name') == 'debug request':
pos = i
break
if pos is not None:
del j[pos]
with open('$FILTRON_RULES', 'w') as rules:
json.dump(j, rules, indent=2, sort_keys=True)
EOF
systemctl restart "${SERVICE_NAME}.service"
}
install_apache_site() {
rst_title "Install Apache site $APACHE_FILTRON_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
! apache_is_installed && info_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_apache
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_apache_site() {
rst_title "Remove Apache site $APACHE_FILTRON_SITE"
@ -551,35 +98,6 @@ This removes apache site ${APACHE_FILTRON_SITE}."
}
install_nginx_site() {
rst_title "Install nginx site $NGINX_FILTRON_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
! nginx_is_installed && info_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_nginx
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
# shellcheck disable=SC2034
SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
# shellcheck disable=SC2034
SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH)
nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_nginx_site() {
rst_title "Remove nginx site $NGINX_FILTRON_SITE"
@ -593,37 +111,10 @@ This removes nginx site ${NGINX_FILTRON_SITE}."
return
fi
nginx_remove_site "$FILTRON_FILTRON_SITE"
nginx_remove_app "$FILTRON_FILTRON_SITE"
}
rst-doc() {
eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""
echo -e "\n.. START install systemd unit"
cat <<EOF
.. tabs::
.. group-tab:: systemd
.. code:: bash
EOF
eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
echo -e "\n.. END install systemd unit"
# for DIST_NAME in ubuntu-20.04 arch fedora centos; do
# (
# DIST_ID=${DIST_NAME%-*}
# DIST_VERS=${DIST_NAME#*-}
# [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
# # ...
# )
# done
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

45
utils/lib.sh
View File

@ -195,7 +195,7 @@ wait_key(){
[[ -n $_t ]] && _t="-t $_t"
printf "$msg"
# shellcheck disable=SC2086
read -r -s -n1 $_t
read -r -s -n1 $_t || true
echo
clean_stdin
}
@ -1117,7 +1117,7 @@ apache_distro_setup() {
APACHE_SITES_AVAILABLE="/etc/httpd/sites-available"
APACHE_SITES_ENABLED="/etc/httpd/sites-enabled"
APACHE_MODULES="modules"
APACHE_PACKAGES="httpd"
APACHE_PACKAGES="httpd mod_ssl"
;;
*)
err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented"
@ -1249,8 +1249,6 @@ apache_dissable_site() {
# -----
uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}"
uWSGI_USER=
uWSGI_GROUP=
# How distros manage uWSGI apps is very different. From uWSGI POV read:
# - https://uwsgi-docs.readthedocs.io/en/latest/Management.html
@ -1276,13 +1274,14 @@ uWSGI_distro_setup() {
;;
fedora-*|centos-7)
# systemd --> /usr/lib/systemd/system/uwsgi.service
# The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see
# - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html
# Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the
# Emperor will run the vassal using the UID/GID of the vassal
# configuration file [1] (user and group of the app .ini file).
# There are some quirks abbout additional POSIX groups in uWSGI
# 2.0.x, read at least: https://github.com/unbit/uwsgi/issues/2099
uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d"
uWSGI_PACKAGES="uwsgi"
uWSGI_USER="uwsgi"
uWSGI_GROUP="uwsgi"
;;
*)
err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
@ -1344,30 +1343,6 @@ uWSGI_restart() {
esac
}
uWSGI_prepare_app() {
# usage: uWSGI_prepare_app <myapp.ini>
[[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>"
local APP="${1%.*}"
case $DIST_ID-$DIST_VERS in
fedora-*|centos-7)
# in emperor mode, the uwsgi user is the owner of the sockets
info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}"
mkdir -p "/run/uwsgi/app/${APP}"
chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}"
;;
*)
info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}"
mkdir -p "/run/uwsgi/app/${APP}"
chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}"
;;
esac
}
uWSGI_app_available() {
# usage: uWSGI_app_available <myapp.ini>
local CONF="$1"
@ -1378,7 +1353,7 @@ uWSGI_app_available() {
uWSGI_install_app() {
# usage: uWSGI_install_app [<template option> ...] <myapp.ini>
# usage: uWSGI_install_app [<template option> ...] <myapp.ini> [{owner} [{group} [{chmod}]]]
#
# <template option>: see install_template
@ -1390,11 +1365,10 @@ uWSGI_install_app() {
*) pos_args+=("$i");;
esac
done
uWSGI_prepare_app "${pos_args[1]}"
mkdir -p "${uWSGI_APPS_AVAILABLE}"
install_template "${template_opts[@]}" \
"${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
root root 644
"${pos_args[2]:-root}" "${pos_args[3]:-root}" "${pos_args[4]:-644}"
uWSGI_enable_app "${pos_args[1]}"
uWSGI_restart "${pos_args[1]}"
info_msg "uWSGI app: ${pos_args[1]} is installed"
@ -1468,7 +1442,6 @@ uWSGI_enable_app() {
mkdir -p "${uWSGI_APPS_ENABLED}"
rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}"
info_msg "enabled uWSGI app: ${CONF}"
;;
*)

207
utils/lib_install.sh
View File

@ -1,207 +0,0 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285
# shellcheck source=utils/lib.sh
. /dev/null
# Initialize installation procedures:
#
# - Modified source_dot_config function that
# - loads .config.sh from an existing installation (at SEARX_SRC).
# - initialize **SEARX_SRC_INIT_FILES**
# - functions like:
# - install_log_searx_instance()
# - install_searx_get_state()
#
# usage:
# source lib_install.sh
#
# **Installation scripts**
#
# The utils/lib_install.sh is sourced by the installations scripts:
#
# - utils/searx.sh
# - utils/morty.sh
# - utils/filtron.sh
#
# If '${SEARX_SRC}/.config.sh' exists, the modified source_dot_config() function
# loads this configuration (instead of './.config.sh').
# **SEARX_SRC_INIT_FILES**
#
# Array of file names to sync into a installation at $SEARX_SRC. The file names
# are relative to the $REPO_ROOT. Set by function init_SEARX_SRC_INIT_FILES().
# Most often theses are files like:
# - .config.sh
# - searx/settings.yml
# - utils/brand.env
# - ...
SEARX_SRC_INIT_FILES=()
eval orig_"$(declare -f source_dot_config)"
source_dot_config() {
# Modified source_dot_config function that
# - loads .config.sh from an existing installation (at SEARX_SRC).
# - initialize SEARX_SRC_INIT_FILES
if [ -z "$eval_SEARX_SRC" ]; then
export eval_SEARX_SRC='true'
SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
SEARX_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_PYENV)
SEARXNG_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_SETTINGS_PATH)
if [ ! -r "${SEARX_SRC}" ]; then
info_msg "not yet cloned: ${SEARX_SRC}"
orig_source_dot_config
return 0
fi
info_msg "using instance at: ${SEARX_SRC}"
# set and log DOT_CONFIG
if [ -r "${SEARX_SRC}/.config.sh" ]; then
info_msg "switching to ${SEARX_SRC}/.config.sh"
DOT_CONFIG="${SEARX_SRC}/.config.sh"
else
info_msg "using local config: ${DOT_CONFIG}"
fi
init_SEARX_SRC_INIT_FILES
fi
}
init_SEARX_SRC_INIT_FILES(){
# init environment SEARX_SRC_INIT_FILES
# Monitor modified files in the working-tree from the local repository, only
# if the local file differs to the corresponding file in the instance. Most
# often theses are files like:
#
# - .config.sh
# - searx/settings.yml
# - utils/brand.env
# - ...
# keep list empty if there is no installation
SEARX_SRC_INIT_FILES=()
if [ ! -r "$SEARX_SRC" ]; then
return 0
fi
local fname
local msg=""
local _prefix=""
if [[ -n ${SUDO_USER} ]]; then
_prefix="sudo -u ${SUDO_USER}"
fi
# Monitor local modified files from the repository, only if the local file
# differs to the corresponding file in the instance
while IFS= read -r fname; do
if [ -z "$fname" ]; then
continue
fi
if [ -r "${SEARX_SRC}/${fname}" ]; then
# diff "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"
if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARX_SRC}/${fname}"; then
SEARX_SRC_INIT_FILES+=("${fname}")
info_msg "local clone (workingtree), modified file: ./$fname"
msg="to update use: sudo -H ./utils/searx.sh install init-src"
fi
fi
done <<< "$($_prefix git diff --name-only)"
[ -n "$msg" ] && info_msg "$msg"
}
install_log_searx_instance() {
echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}"
echo -e " SEARXNG_SETTINGS_PATH : ${_BBlue}${SEARXNG_SETTINGS_PATH}${_creset}"
echo -e " SEARX_PYENV : ${_BBlue}${SEARX_PYENV}${_creset}"
echo -e " SEARX_SRC : ${_BBlue}${SEARX_SRC:-none}${_creset}"
echo -e " SEARXNG_URL : ${_BBlue}${SEARXNG_URL:-none}${_creset}"
if in_container; then
# SearXNG is listening on 127.0.0.1 and not available from outside container
# in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
echo -e "---- container setup"
echo -e " ${_BBlack}HINT:${_creset} SearXNG only listen on loopback device" \
"${_BBlack}inside${_creset} the container."
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
echo " container (IPv6): [${ip#*|}]"
else
# IPv4:
echo " container (IPv4): ${ip#*|}"
fi
done
fi
}
install_searx_get_state(){
# usage: install_searx_get_state
#
# Prompts a string indicating the status of the installation procedure
#
# missing-searx-clone:
# There is no clone at ${SEARX_SRC}
# missing-searx-pyenv:
# There is no pyenv in ${SEARX_PYENV}
# installer-modified:
# There are files modified locally in the installer (clone),
# see ${SEARX_SRC_INIT_FILES} description.
# python-installed:
# Scripts can be executed in instance's environment
# - user: ${SERVICE_USER}
# - pyenv: ${SEARX_PYENV}
if [ -f /etc/searx/settings.yml ]; then
err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/"
fi
if ! [ -r "${SEARX_SRC}" ]; then
echo "missing-searx-clone"
return
fi
if ! [ -f "${SEARX_PYENV}/bin/activate" ]; then
echo "missing-searx-pyenv"
return
fi
if ! [ -r "${SEARXNG_SETTINGS_PATH}" ]; then
echo "missing-settings"
return
fi
if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then
echo "installer-modified"
return
fi
echo "python-installed"
}
# Initialization of the installation procedure
# --------------------------------------------
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"
# SEARXNG_URL aka PUBLIC_URL: the public URL of the instance (e.g.
# "https://example.org/searx"). The value is taken from environment $SEARXNG_URL
# in ./utils/brand.env. This variable is a empty string if server.base_url in
# the settings.yml is set to 'false'.
SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)}"
if in_container; then
# hint: Linux containers do not have DNS entries, lets use IPs
SEARXNG_URL="http://$(primary_ip)"
fi
PUBLIC_URL="${SEARXNG_URL}"
source_dot_config
# shellcheck source=utils/lxc-searx.env
source "${REPO_ROOT}/utils/lxc-searx.env"
in_container && lxc_set_suite_env

21
utils/lib_redis.sh
View File

@ -42,6 +42,8 @@ REDIS_GIT_URL="https://github.com/redis/redis.git"
REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}"
REDIS_USER="searxng-redis"
REDIS_GROUP="searxng-redis"
REDIS_HOME="/usr/local/${REDIS_USER}"
REDIS_HOME_BIN="${REDIS_HOME}/.local/bin"
REDIS_ENV="${REDIS_HOME}/.redis_env"
@ -113,7 +115,7 @@ redis.devpkg() {
case ${DIST_ID} in
ubuntu|debian)
pkg_install git build-essential
pkg_install git build-essential gawk
;;
arch)
pkg_install git base-devel
@ -140,14 +142,19 @@ redis.build() {
redis.src "${CACHE}/redis"
if ! required_commands gcc nm make gawk ; then
info_msg "install development tools to get missing command(s) .."
if [[ -n ${SUDO_USER} ]]; then
sudo -H "$0" redis.devpkg
else
redis.devpkg
fi
fi
rst_title "compile redis sources" section
pushd "${CACHE}/redis" &>/dev/null
if ask_yn "Do you run 'make distclean' first'?" Ny; then
if ask_yn "Do you run 'make distclean' first'?" Yn; then
$(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout
fi
@ -233,7 +240,7 @@ useradd --shell /bin/bash --system \
--home-dir "${REDIS_HOME}" \
--comment 'user that runs a redis instance' "${REDIS_USER}"
mkdir -p "${REDIS_HOME}"
chown -R "${REDIS_USER}:${REDIS_USER}" "${REDIS_HOME}"
chown -R "${REDIS_USER}:${REDIS_GROUP}" "${REDIS_HOME}"
groups "${REDIS_USER}"
EOF
@ -248,7 +255,7 @@ EOF
redis.userdel() {
sudo_or_exit
drop_service_account "${REDIS_USER}"
groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true
groupdel "${REDIS_GROUP}" 2>&1 | prefix_stdout || true
}
redis.addgrp() {
@ -256,7 +263,7 @@ redis.addgrp() {
# usage: redis.addgrp <user>
[[ -z $1 ]] && die_caller 42 "missing argument <user>"
sudo -H gpasswd -a "$1" "${REDIS_USER}"
sudo -H gpasswd -a "$1" "${REDIS_GROUP}"
}
redis.rmgrp() {
@ -264,7 +271,7 @@ redis.rmgrp() {
# usage: redis.rmgrp <user>
[[ -z $1 ]] && die_caller 42 "missing argument <user>"
sudo -H gpasswd -d "$1" "${REDIS_USER}"
sudo -H gpasswd -d "$1" "${REDIS_GROUP}"
}
@ -278,7 +285,7 @@ redis._install_bin() {
(
set -e
for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do
install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \
install -v -o "${REDIS_USER}" -g "${REDIS_GROUP}" \
"${src}/${redis_exe}" "${REDIS_HOME_BIN}"
done

49
utils/lxc-searx.env → utils/lxc-searxng.env
View File

@ -4,24 +4,18 @@
# This file is a setup of a LXC suite. It is sourced from different context, do
# not manipulate the environment directly, implement functions and manipulate
# environment only is subshells!
# environment only in subshells.
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
# shellcheck disable=SC2034
LXC_SUITE_NAME="searx"
lxc_set_suite_env() {
export LXC_SUITE_NAME="searxng"
# name of https://images.linuxcontainers.org
export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}"
export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}"
export LXC_SUITE=(
# to disable containers, comment out lines ..
# end of standard support see https://wiki.ubuntu.com/Releases
"$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023
"$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # April 2025
"$LINUXCONTAINERS_ORG_NAME:ubuntu/21.10" "ubu2110" # July 2027
@ -30,36 +24,17 @@ lxc_set_suite_env() {
# rolling releases see https://www.archlinux.org/releng/releases/
"$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux"
# EOL 30 June 2024
"$LINUXCONTAINERS_ORG_NAME:centos/7" "centos7"
)
PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"
if in_container; then
# container hostnames do not have a DNS entry: use primary IP!
PUBLIC_URL="http://$(primary_ip)/searx"
# make GUEST's services public to the HOST
FILTRON_API="0.0.0.0:4005"
FILTRON_LISTEN="0.0.0.0:4004"
MORTY_LISTEN="0.0.0.0:3000"
# export LXC specific environment
export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN
fi
}
lxc_suite_install_info() {
(
lxc_set_suite_env
cat <<EOF
LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL}
suite includes searx, morty & filtron
suite images:
$(echo " ${LOCAL_IMAGES[*]}" | $FMT)
suite containers:
$(echo " ${CONTAINERS[*]}" | $FMT)
LXC suite: ${LXC_SUITE_NAME}
Suite includes installation of SearXNG
images: ${LOCAL_IMAGES[*]}
containers: ${CONTAINERS[*]}
EOF
)
}
@ -69,10 +44,7 @@ lxc_suite_install() {
lxc_set_suite_env
FORCE_TIMEOUT=0
export FORCE_TIMEOUT
"${LXC_REPO_ROOT}/utils/searx.sh" install all
"${LXC_REPO_ROOT}/utils/morty.sh" install all
"${LXC_REPO_ROOT}/utils/filtron.sh" install all
"${LXC_REPO_ROOT}/utils/searxng.sh" install all
rst_title "suite installation finished ($(hostname))" part
lxc_suite_info
echo
@ -88,10 +60,9 @@ lxc_suite_info() {
else
# IPv4:
# shellcheck disable=SC2034,SC2031
info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL"
info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY"
info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/"
fi
done
"${LXC_REPO_ROOT}/utils/searxng.sh" searxng.instance.env
)
}

3
utils/lxc.sh
View File

@ -4,12 +4,11 @@
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
source_dot_config
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"
# load environment of the LXC suite
LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}"
LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searxng.env}"
source "$LXC_ENV"
lxc_set_suite_env

457
utils/morty.sh
View File

@ -3,10 +3,6 @@
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/lib_go.sh
source "${REPO_ROOT}/utils/lib_go.sh"
# shellcheck source=utils/lib_install.sh
source "${REPO_ROOT}/utils/lib_install.sh"
# ----------------------------------------------------------------------------
# config
@ -16,24 +12,9 @@ MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}"
PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}"
# shellcheck disable=SC2034
MORTY_TIMEOUT=5
SERVICE_NAME="morty"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
# shellcheck disable=SC2034
SERVICE_ENV_DEBUG=false
GO_ENV="${SERVICE_HOME}/.go_env"
GO_VERSION="go1.17.2"
# shellcheck disable=SC2034
CONFIG_FILES=()
# Apache Settings
@ -47,267 +28,45 @@ usage() {
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") shell
$(basename "$0") install [all|check|user]
$(basename "$0") reinstall all
$(basename "$0") update [morty]
$(basename "$0") remove [all]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-on|debug-off|new-key]
$(basename "$0") apache [install|remove]
$(basename "$0") nginx [install|remove]
$(basename "$0") info [searx]
$(basename "$0") remove all
$(basename "$0") apache remove
$(basename "$0") nginx remove
shell
start interactive shell from user ${SERVICE_USER}
install / remove
:all: complete setup of morty service
:user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
install
:check: check the morty installation
reinstall:
:all: runs 'install/remove all'
update morty
Update morty installation ($SERVICE_HOME)
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
show service status and log
option
set one of the available options
:new-key: set new morty key
apache : ${PUBLIC_URL_MORTY}
:install: apache site with a reverse proxy (ProxyPass)
:remove: apache site ${APACHE_MORTY_SITE}
nginx (${PUBLIC_URL_MORTY})
:install: nginx site with a reverse proxy (ProxyPass)
:remove: nginx site ${NGINX_MORTY_SITE}
----
sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY}
MORTY_LISTEN: : ${MORTY_LISTEN}
remove all : drop all components of the morty service
apache remove : drop apache site ${APACHE_MORTY_SITE}
nginx remove : drop nginx site ${NGINX_MORTY_SITE}
EOF
install_log_searx_instance
if in_container; then
# in containers the service is listening on 0.0.0.0 (see lxc-searx.env)
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
echo " container URL (IPv6): http://[${ip#*|}]:3000/"
else
# IPv4:
echo " container URL (IPv4): http://${ip#*|}:3000/"
fi
done
fi
echo
info_searx
[[ -n ${1} ]] && err_msg "$1"
}
info_searx() {
# shellcheck disable=SC1117
cat <<EOF
To activate result and image proxy in SearXNG read:
https://docs.searxng.org/admin/morty.html
Check settings in file ${SEARXNG_SETTINGS_PATH} ...
result_proxy:
url : ${PUBLIC_URL_MORTY}
server:
image_proxy : True
EOF
}
main() {
required_commands \
sudo install git wget curl \
|| exit
local _usage="ERROR: unknown or missing $1 command $2"
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
shell)
sudo_or_exit
interactive_shell "${SERVICE_USER}"
;;
inspect)
case $2 in
service)
sudo_or_exit
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
reinstall)
rst_title "re-install $SERVICE_NAME" part
sudo_or_exit
case $2 in
all)
remove_all
install_all
;;
*) usage "$_usage"; exit 42;;
esac ;;
install)
rst_title "$SERVICE_NAME" part
sudo_or_exit
case $2 in
all) install_all ;;
check)
rst_title "Check morty installation" part
install_check
;;
user) assert_user ;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
sudo_or_exit
case $2 in
morty) update_morty ;;
*) usage "$_usage"; exit 42;;
esac ;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
user) drop_service_account "${SERVICE_USER}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
activate)
sudo_or_exit
case $2 in
service) systemd_activate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) systemd_deactivate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
nginx)
sudo_or_exit
case $2 in
install) install_nginx_site ;;
remove) remove_nginx_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
info)
case $2 in
searx) info_searx ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
sudo_or_exit
case $2 in
new-key) set_new_key ;;
debug-on) enable_debug ;;
debug-off) disable_debug ;;
*) usage "$_usage"; exit 42;;
esac ;;
doc) rst-doc ;;
*) usage "ERROR: unknown or missing command $1"; exit 42;;
esac
}
install_all() {
MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
rst_title "Install $SERVICE_NAME (service)"
assert_user
wait_key
go.golang "${GO_VERSION}" "${SERVICE_USER}"
wait_key
install_morty
wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
wait_key
if ! service_is_available "http://${MORTY_LISTEN}" ; then
err_msg "Morty is not listening on: http://${MORTY_LISTEN}"
fi
if apache_is_installed; then
info_msg "Apache is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_apache_site
fi
elif nginx_is_installed; then
info_msg "nginx is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_nginx_site
fi
fi
info_searx
if ask_yn "Add image and result proxy to SearXNG settings.yml?" Yn; then
"${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
"${REPO_ROOT}/utils/searx.sh" option image-proxy-on
fi
if ask_yn "Do you want to inspect the installation?" Ny; then
inspect_service
fi
}
install_check() {
if service_account_is_available "$SERVICE_USER"; then
info_msg "service account $SERVICE_USER available."
else
err_msg "service account $SERVICE_USER not available!"
fi
if go_is_available "$SERVICE_USER"; then
info_msg "~$SERVICE_USER: go is installed"
else
err_msg "~$SERVICE_USER: go is not installed"
fi
if morty_is_installed; then
info_msg "~$SERVICE_USER: morty app is installed"
else
err_msg "~$SERVICE_USER: morty app is not installed!"
fi
if ! service_is_available "http://${MORTY_LISTEN}" ; then
err_msg "Morty is not listening on: http://${MORTY_LISTEN}"
echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .."
wait_key
fi
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
if ! in_container; then
warn_msg "Check if public name is correct and routed or use the public IP from above."
fi
fi
if [[ "${GO_VERSION}" > "$(go_version)" ]]; then
warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least"
warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all"
else
info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)"
fi
}
go_version(){
go.version "${SERVICE_USER}"
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
@ -321,152 +80,6 @@ installations that were installed with this script."
fi
}
assert_user() {
rst_title "user $SERVICE_USER" section
echo
tee_stderr 1 <<EOF | bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "$SERVICE_HOME" \
--comment 'Web content sanitizer proxy' $SERVICE_USER
mkdir "$SERVICE_HOME"
chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
groups $SERVICE_USER
EOF
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
export SERVICE_HOME
echo "export SERVICE_HOME=$SERVICE_HOME"
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
touch $GO_ENV
grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile
EOF
}
morty_is_installed() {
[[ -f $SERVICE_HOME/go-apps/bin/morty ]]
}
install_morty() {
rst_title "Install morty in user's ~/go-apps" section
echo
go.install github.com/asciimoo/morty@latest "${SERVICE_USER}"
}
update_morty() {
rst_title "Update morty" section
echo
go.install github.com/asciimoo/morty@latest "${SERVICE_USER}"
}
set_service_env_debug() {
# usage: set_service_env_debug [false|true]
# shellcheck disable=SC2034
local SERVICE_ENV_DEBUG="${1:-false}"
if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
fi
}
inspect_service() {
rst_title "service status & log"
cat <<EOF
sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY}
MORTY_LISTEN: : ${MORTY_LISTEN}
EOF
install_log_searx_instance
install_check
if in_container; then
lxc_suite_info
else
info_msg "public URL --> ${PUBLIC_URL_MORTY}"
info_msg "morty URL --> http://${MORTY_LISTEN}"
fi
local _debug_on
if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then
enable_debug
_debug_on=1
else
systemctl --no-pager -l status "${SERVICE_NAME}"
fi
echo
# shellcheck disable=SC2059
printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
read -r -s -n1 -t 5
echo
while true; do
trap break 2
journalctl -f -u "${SERVICE_NAME}"
done
if [[ $_debug_on == 1 ]]; then
FORCE_SELECTION=Y disable_debug
fi
return 0
}
enable_debug() {
warn_msg "Do not enable debug in production environments!!"
info_msg "Enabling debug option needs to reinstall systemd service!"
set_service_env_debug true
}
disable_debug() {
info_msg "Disabling debug option needs to reinstall systemd service!"
set_service_env_debug false
}
set_new_key() {
rst_title "Set morty key"
echo
MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
info_msg "morty key: '${MORTY_KEY}'"
warn_msg "this will need to reinstall services .."
MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
"${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
"${REPO_ROOT}/utils/searx.sh" option image-proxy-on
}
install_apache_site() {
rst_title "Install Apache site $APACHE_MORTY_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})"
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_apache
fi
apache_install_site "${APACHE_MORTY_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
fi
}
remove_apache_site() {
@ -484,35 +97,6 @@ This removes apache site ${APACHE_MORTY_SITE}."
apache_remove_site "$APACHE_MORTY_SITE"
}
install_nginx_site() {
rst_title "Install nginx site $NGINX_MORTY_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})"
! nginx_is_installed && err_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_nginx
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
# shellcheck disable=SC2034
SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
# shellcheck disable=SC2034
SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH)
nginx_install_app "${NGINX_MORTY_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
fi
}
remove_nginx_site() {
rst_title "Remove nginx site $NGINX_MORTY_SITE"
@ -526,37 +110,10 @@ This removes nginx site ${NGINX_MORTY_SITE}."
return
fi
nginx_remove_site "$NGINX_MORTY_SITE"
nginx_remove_app "$NGINX_MORTY_SITE"
}
rst-doc() {
eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\""
echo -e "\n.. START install systemd unit"
cat <<EOF
.. tabs::
.. group-tab:: systemd
.. code:: bash
EOF
eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
echo -e "\n.. END install systemd unit"
# for DIST_NAME in ubuntu-20.04 arch fedora centos; do
# (
# DIST_ID=${DIST_NAME%-*}
# DIST_VERS=${DIST_NAME#*-}
# [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
# # ...
# )
# done
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

1031
utils/searx.sh
View File

File diff suppressed because it is too large Load Diff

1017
utils/searxng.sh Executable file
View File

File diff suppressed because it is too large Load Diff

7
utils/searxng_check.py
View File

@ -25,3 +25,10 @@ if os.path.isfile(OLD_SETTING):
os.environ.get('SEARXNG_SETTINGS_PATH', '/etc/searxng/settings.yml')
))
warnings.warn(msg, DeprecationWarning)
from searx.shared import redisdb
from searx import get_setting
if not redisdb.init():
warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2)
warnings.warn("--> no bot protection without redis DB", RuntimeWarning, stacklevel=2)

129
utils/templates/etc/filtron/rules.json
View File

@ -1,129 +0,0 @@
[
{
"name": "roboagent limit",
"filters": [
"Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)"
],
"limit": 0,
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "botlimit",
"filters": [
"Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
],
"limit": 0,
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "suspiciously frequent IP",
"filters": [],
"interval": 600,
"limit": 30,
"aggregations": [
"Header:X-Forwarded-For"
],
"actions":[
{"name":"log"}
]
},
{
"name": "search request",
"filters": [
"Param:q",
"Path=^(/|/search)$"
],
"interval": 61,
"limit": 999,
"subrules": [
{
"name": "missing Accept-Language",
"filters": ["!Header:Accept-Language"],
"limit": 0,
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "suspiciously Connection=close header",
"filters": ["Header:Connection=close"],
"limit": 0,
"stop": true,
"actions": [
{"name":"log"},
{"name": "block",
"params": {"message": "Rate limit exceeded"}}
]
},
{
"name": "IP limit",
"interval": 61,
"limit": 9,
"stop": true,
"aggregations": [
"Header:X-Forwarded-For"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "rss/json limit",
"filters": [
"Param:format=(csv|json|rss)"
],
"interval": 121,
"limit": 2,
"stop": true,
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
},
{
"name": "useragent limit",
"interval": 61,
"limit": 199,
"aggregations": [
"Header:User-Agent"
],
"actions": [
{ "name": "log"},
{ "name": "block",
"params": {
"message": "Rate limit exceeded"
}
}
]
}
]
}
]

28
utils/templates/etc/httpd/sites-available/morty.conf
View File

@ -1,28 +0,0 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog
# CustomLog /dev/null combined env=dontlog
<Location ${PUBLIC_URL_PATH_MORTY} >
<IfModule mod_security2.c>
SecRuleEngine Off
</IfModule>
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass http://${MORTY_LISTEN}
RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY}
</Location>

41
utils/templates/etc/httpd/sites-available/searxng.conf Normal file
View File

@ -0,0 +1,41 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
#
# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog
# CustomLog /dev/null combined env=dontlog
<Location ${SEARXNG_URL_PATH}>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
# add the trailing slash
RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/
ProxyPreserveHost On
ProxyPass http://${SEARXNG_INTERNAL_HTTP}
# see flaskfix.py
RequestHeader set X-Scheme %{REQUEST_SCHEME}s
RequestHeader set X-Script-Name ${SEARXNG_URL_PATH}
# see limiter.py
RequestHeader set X-Real-IP %{REMOTE_ADDR}s
RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s
</Location>
# uWSGI serves the static files and in settings.yml we use::
#
# ui:
# static_use_hash: true
#
# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/

33
utils/templates/etc/httpd/sites-available/searxng.conf:filtron
View File

@ -1,33 +0,0 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so
#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog
# CustomLog /dev/null combined env=dontlog
# SecRuleRemoveById 981054
# SecRuleRemoveById 981059
# SecRuleRemoveById 981060
# SecRuleRemoveById 950907
<Location ${FILTRON_URL_PATH} >
<IfModule mod_security2.c>
SecRuleEngine Off
</IfModule>
Require all granted
Order deny,allow
Deny from all
#Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass http://${FILTRON_LISTEN}
RequestHeader set X-Script-Name ${FILTRON_URL_PATH}
</Location>

41
utils/templates/etc/httpd/sites-available/searxng.conf:socket Normal file
View File

@ -0,0 +1,41 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so
# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
#
# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog
# CustomLog /dev/null combined env=dontlog
<Location ${SEARXNG_URL_PATH}>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
# add the trailing slash
RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/
ProxyPreserveHost On
ProxyPass unix:${SEARXNG_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searxng/
# see flaskfix.py
RequestHeader set X-Scheme %{REQUEST_SCHEME}s
RequestHeader set X-Script-Name ${SEARXNG_URL_PATH}
# see limiter.py
RequestHeader set X-Real-IP %{REMOTE_ADDR}s
RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s
</Location>
# uWSGI serves the static files and in settings.yml we use::
#
# ui:
# static_use_hash: true
#
# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/

27
utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi
View File

@ -1,27 +0,0 @@
# -*- coding: utf-8; mode: apache -*-
LoadModule headers_module ${APACHE_MODULES}/mod_headers.so
LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so
LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so
# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so
# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog
# CustomLog /dev/null combined env=dontlog
<Location ${SEARXNG_URL_PATH}>
<IfModule mod_security2.c>
SecRuleEngine Off
</IfModule>
Require all granted
Order deny,allow
Deny from all
# Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1
Allow from all
ProxyPreserveHost On
ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/
</Location>

11
utils/templates/etc/nginx/default.apps-available/morty.conf
View File

@ -1,11 +0,0 @@
# https://example.org/morty
location /morty {
proxy_pass http://127.0.0.1:3000/;
proxy_set_header Host \$host;
proxy_set_header Connection \$http_connection;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Scheme \$scheme;
}

29
utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron → utils/templates/etc/nginx/default.apps-available/searxng.conf
View File

@ -1,16 +1,29 @@
# https://example.org/searx
location ${SEARXNG_URL_PATH} {
proxy_pass http://127.0.0.1:4004/;
proxy_pass http://${SEARXNG_INTERNAL_HTTP};
proxy_set_header Host \$host;
proxy_set_header Connection \$http_connection;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
# see flaskfix.py
proxy_set_header X-Scheme \$scheme;
proxy_set_header X-Script-Name ${SEARXNG_URL_PATH};
# see limiter.py
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
# proxy_buffering off;
# proxy_request_buffering off;
# proxy_buffer_size 8k;
}
location ${SEARXNG_URL_PATH}/static/ {
alias ${SEARX_SRC}/searx/static/;
}
# uWSGI serves the static files and in settings.yml we use::
#
# ui:
# static_use_hash: true
#
# location ${SEARXNG_URL_PATH}/static/ {
# alias ${SEARXNG_STATIC}/;
# }

26
utils/templates/etc/nginx/default.apps-available/searxng.conf:socket Normal file
View File

@ -0,0 +1,26 @@
location ${SEARXNG_URL_PATH} {
uwsgi_pass unix://${SEARXNG_UWSGI_SOCKET};
include uwsgi_params;
uwsgi_param HTTP_HOST \$host;
uwsgi_param HTTP_CONNECTION \$http_connection;
# see flaskfix.py
uwsgi_param HTTP_X_SCHEME \$scheme;
uwsgi_param HTTP_X_SCRIPT_NAME ${SEARXNG_URL_PATH};
# see limiter.py
uwsgi_param HTTP_X_REAL_IP \$remote_addr;
uwsgi_param HTTP_X_FORWARDED_FOR \$proxy_add_x_forwarded_for;
}
# uWSGI serves the static files and in settings.yml we use::
#
# ui:
# static_use_hash: true
#
# location ${SEARXNG_URL_PATH}/static/ {
# alias ${SEARXNG_STATIC}/;
# }

78
utils/templates/etc/searxng/settings.yml
View File

@ -1,46 +1,55 @@
# SearXNG settings, before editing this file read:
#
# https://docs.searxng.org/admin/engines/settings.html
# SearXNG settings
use_default_settings: true
general:
# Debug mode, only for development
debug: false
# change displayed name
# instance_name: "SearXNG"
instance_name: "SearXNG"
search:
# Filter results. 0: None, 1: Moderate, 2: Strict
safe_search: 0
# Existing autocomplete backends: "dbpedia", "duckduckgo", "google",
# "startpage", "swisscows", "qwant", "wikipedia" - leave blank to turn it off
# by default.
autocomplete: ''
# Default search language - leave blank to detect from browser information or
# use codes from 'languages.py'
default_lang: ''
# remove format to deny access, use lower case.
formats:
- html
safe_search: 2
autocomplete: 'duckduckgo'
server:
secret_key: "ultrasecretkey" # change this!
# Proxying image results through SearXNG
image_proxy: false
secret_key: "ultrasecretkey"
limiter: true
image_proxy: true
# result_proxy:
# url: http://127.0.0.1:3000/
# key: !!binary "your_morty_proxy_key"
redis:
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
ui:
static_use_hash: true
# preferences:
# lock:
# - autocomplete
# - method
enabled_plugins:
- 'Hash plugin'
- 'Search on category select'
- 'Self Informations'
- 'Tracker URL remover'
- 'Ahmia blacklist'
# - 'Hostname replace' # see hostname_replace configuration below
# - 'Infinite scroll'
# - 'Open Access DOI rewrite'
# - 'Vim-like hotkeys'
# plugins:
# - only_show_green_results
# engines:
#
# - name: duckduckgo
# disabled: false
# hostname_replace:
#
# # twitter --> nitter
# '(www\.)?twitter\.com$': 'nitter.net'
engines:
- name: google
use_mobile_ui: true
# - name: fdroid
# disabled: false
#
@ -48,6 +57,13 @@ server:
# disabled: false
#
# - name: mediathekviewweb
# engine: mediathekviewweb
# shortcut: mvw
# categories: general
# categories: TV
# disabled: false
#
# - name: invidious
# disabled: false
# base_url:
# - https://invidious.snopyta.org
# - https://invidious.tiekoetter.com
# - https://invidio.xamh.de
# - https://inv.riverside.rocks

26
utils/templates/etc/uwsgi/apps-archlinux/searxng.ini
View File

@ -16,7 +16,7 @@ env = LANGUAGE=C.UTF-8
env = LC_ALL=C.UTF-8
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
chdir = ${SEARXNG_SRC}/searx
# SearXNG configuration (settings.yml)
env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
@ -57,37 +57,27 @@ enable-threads = true
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
virtualenv = ${SEARXNG_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
pythonpath = ${SEARXNG_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
http = ${SEARX_INTERNAL_HTTP}
http = ${SEARXNG_INTERNAL_HTTP}
# using unix-sockets:
# uWSGI serves the static files and in settings.yml we use::
#
# On some distributions you need to create the app folder for the sockets::
# ui:
# static_use_hash: true
#
# mkdir -p ${SEARX_UWSGI_SOCKET}
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET}
#
# socket = ${SEARX_UWSGI_SOCKET}
# uwsgi serves the static files
static-map = /static=${SEARXNG_STATIC}
# expires set to one year since there are hashes
static-map = /static=${SEARX_SRC}/searx/static
static-expires = /* 31557600
static-gzip-all = True
offload-threads = %k

29
utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket
View File

@ -16,7 +16,7 @@ env = LANGUAGE=C.UTF-8
env = LC_ALL=C.UTF-8
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
chdir = ${SEARXNG_SRC}/searx
# SearXNG configuration (settings.yml)
env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
@ -57,37 +57,24 @@ enable-threads = true
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
virtualenv = ${SEARXNG_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
pythonpath = ${SEARXNG_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
socket = ${SEARXNG_UWSGI_SOCKET}
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
# uWSGI serves the static files and in settings.yml we use::
#
# On some distributions you need to create the app folder for the sockets::
# ui:
# static_use_hash: true
#
# mkdir -p ${SEARX_UWSGI_SOCKET}
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET}
#
socket = ${SEARX_UWSGI_SOCKET}
# uwsgi serves the static files
static-map = /static=${SEARXNG_STATIC}
# expires set to one year since there are hashes
static-map = /static=${SEARX_SRC}/searx/static
static-expires = /* 31557600
static-gzip-all = True
offload-threads = %k

32
utils/templates/etc/uwsgi/apps-available/searxng.ini
View File

@ -6,7 +6,11 @@
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be
# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini).
#
# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting
#
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
@ -16,7 +20,7 @@ env = LANGUAGE=C.UTF-8
env = LC_ALL=C.UTF-8
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
chdir = ${SEARXNG_SRC}/searx
# SearXNG configuration (settings.yml)
env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
@ -56,37 +60,27 @@ enable-threads = true
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
virtualenv = ${SEARXNG_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
pythonpath = ${SEARXNG_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
http = ${SEARX_INTERNAL_HTTP}
http = ${SEARXNG_INTERNAL_HTTP}
# using unix-sockets:
# uWSGI serves the static files and in settings.yml we use::
#
# On some distributions you need to create the app folder for the sockets::
# ui:
# static_use_hash: true
#
# mkdir -p /run/uwsgi/app/searxng
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET}
#
# socket = ${SEARX_UWSGI_SOCKET}
# uwsgi serves the static files
static-map = /static=${SEARXNG_STATIC}
# expires set to one year since there are hashes
static-map = /static=${SEARX_SRC}/searx/static
static-expires = /* 31557600
static-gzip-all = True
offload-threads = %k

35
utils/templates/etc/uwsgi/apps-available/searxng.ini:socket
View File

@ -6,7 +6,11 @@
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core
# Who will run the code
# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be
# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini).
#
# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting
#
uid = ${SERVICE_USER}
gid = ${SERVICE_GROUP}
@ -16,7 +20,7 @@ env = LANGUAGE=C.UTF-8
env = LC_ALL=C.UTF-8
# chdir to specified directory before apps loading
chdir = ${SEARX_SRC}/searx
chdir = ${SEARXNG_SRC}/searx
# SearXNG configuration (settings.yml)
env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
@ -56,37 +60,24 @@ enable-threads = true
module = searx.webapp
# set PYTHONHOME/virtualenv
virtualenv = ${SEARX_PYENV}
virtualenv = ${SEARXNG_PYENV}
# add directory (or glob) to pythonpath
pythonpath = ${SEARX_SRC}
pythonpath = ${SEARXNG_SRC}
# speak to upstream
# -----------------
#
# Activate the 'http' configuration for filtron or activate the 'socket'
# configuration if you setup your HTTP server to use uWSGI protocol via sockets.
# using IP:
#
# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http
# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html
socket = ${SEARXNG_UWSGI_SOCKET}
# http = ${SEARX_INTERNAL_HTTP}
# using unix-sockets:
# uWSGI serves the static files and in settings.yml we use::
#
# On some distributions you need to create the app folder for the sockets::
# ui:
# static_use_hash: true
#
# mkdir -p ${SEARX_UWSGI_SOCKET}
# chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${SEARX_UWSGI_SOCKET}
#
socket = ${SEARX_UWSGI_SOCKET}
# uwsgi serves the static files
static-map = /static=${SEARXNG_STATIC}
# expires set to one year since there are hashes
static-map = /static=${SEARX_SRC}/searx/static
static-expires = /* 31557600
static-gzip-all = True
offload-threads = %k