Merge pull request #2408 from return42/rm-brand-make

[mod] move brand options from Makefile to settings.yml

Makefile

@@ -1,14 +1,6 @@
 # -*- coding: utf-8; mode: makefile-gmake -*-
 .DEFAULT_GOAL=help
 
-# START Makefile setup
-export GIT_URL=https://github.com/searx/searx
-export GIT_BRANCH=master
-export SEARX_URL=https://searx.me
-export DOCS_URL=https://searx.github.io/searx
-# export CONTACT_URL=mailto:contact@example.com
-# END Makefile setup
-
 include utils/makefile.include
 
 PYOBJECTS = searx
@@ -43,12 +35,6 @@ help-min:
 	@echo  '  docker    - build Docker image'
 	@echo  '  node.env  - download & install npm dependencies locally'
 	@echo  ''
-	@echo  'environment'
-	@echo  '  SEARX_URL = $(SEARX_URL)'
-	@echo  '  GIT_URL   = $(GIT_URL)'
-	@echo  '  DOCS_URL  = $(DOCS_URL)'
-	@echo  '  CONTACT_URL = $(CONTACT_URL)'
-	@echo  ''
 	@$(MAKE) -e -s make-help
 
 help-all: help-min
@@ -118,24 +104,8 @@ useragents.update:  pyenvinstall
 	$(Q)echo "Update searx/data/useragents.json with the most recent versions of Firefox."
 	$(Q)$(PY_ENV_ACT); python utils/fetch_firefox_version.py
 
-buildenv:
+buildenv: pyenv
-	$(Q)echo "build searx/brand.py"
+	$(Q)$(PY_ENV_ACT); SEARX_DEBUG=1 python utils/build_env.py
-	$(Q)echo "GIT_URL = '$(GIT_URL)'"  > searx/brand.py
-	$(Q)echo "GIT_BRANCH = '$(GIT_BRANCH)'"  >> searx/brand.py
-	$(Q)echo "ISSUE_URL = 'https://github.com/searx/searx/issues'" >> searx/brand.py
-	$(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py
-	$(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py
-	$(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py
-	$(Q)echo "CONTACT_URL = '$(CONTACT_URL)'" >> searx/brand.py
-	$(Q)echo "build utils/brand.env"
-	$(Q)echo "export GIT_URL='$(GIT_URL)'"  > utils/brand.env
-	$(Q)echo "export GIT_BRANCH='$(GIT_BRANCH)'"  >> utils/brand.env
-	$(Q)echo "export ISSUE_URL='https://github.com/searx/searx/issues'" >> utils/brand.env
-	$(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env
-	$(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env
-	$(Q)echo "export PUBLIC_INSTANCES='https://searx.space'" >> utils/brand.env
-	$(Q)echo "export CONTACT_URL='$(CONTACT_URL)'" >> utils/brand.env
-
 
 # node / npm
 # ----------

docs/admin/installation-searx.rst

@@ -70,13 +70,20 @@ from the login (*~/.profile*):
 Configuration
 =============
 
+.. sidebar:: ``use_default_settings: True``
+
+   - :ref:`settings global`
+   - :ref:`settings location`
+   - :ref:`settings use_default_settings`
+   - :origin:`/etc/searx/settings.yml <utils/templates/etc/searx/use_default_settings.yml>`
+
 To create a initial ``/etc/searx/settings.yml`` you can start with a copy of the
 file :origin:`utils/templates/etc/searx/use_default_settings.yml`.  This setup
-:option:ref:`use default settings <settings use_default_settings>` from
+:ref:`use default settings <settings use_default_settings>` from
 :origin:`searx/settings.yml` and is recommended since :pull:`2291` is merged.
 
-For minimal Setup, configure like shown below – replace ``searx@\$(uname -n)``
+For a *minimal setup*, configure like shown below – replace ``searx@$(uname
-with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit
+-n)`` with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit
 ``/etc/searx/settings.yml`` to your needs.
 
 .. kernel-include:: $DOCS_BUILD/includes/searx.rst

docs/admin/installation.rst

@@ -76,6 +76,6 @@ If all services are running fine, you can add it to your HTTP server:
 .. tip::
 
    About script's installation options have a look at chapter :ref:`toolboxing
-   setup`.  How to brand your instance see chapter :ref:`makefile setup`.  To
+   setup`.  How to brand your instance see chapter :ref:`settings global`.  To
    *stash* your instance's setup, `git stash`_ your clone's :origin:`Makefile`
    and :origin:`.config.sh` file .

docs/admin/settings.rst

@@ -27,7 +27,8 @@ First, searx will try to load settings.yml from these locations:
 1. the full path specified in the ``SEARX_SETTINGS_PATH`` environment variable.
 2. ``/etc/searx/settings.yml``
 
-If these files don't exist (or are empty or can't be read), searx uses the :origin:`searx/settings.yml` file.
+If these files don't exist (or are empty or can't be read), searx uses the
+:origin:`searx/settings.yml` file.
 
 
 .. _settings global:
@@ -35,16 +36,46 @@ If these files don't exist (or are empty or can't be read), searx uses the :orig
 Global Settings
 ===============
 
+``general:``
+------------
+
 .. code:: yaml
 
    general:
        debug : False # Debug mode, only for development
        instance_name : "searx" # displayed name
+       git_url: https://github.com/searx/searx
+       git_branch: master
+       issue_url: https://github.com/searx/searx/issues
+       docs_url: https://searx.github.io/searx
+       public_instances: https://searx.space
+       contact_url: False # mailto:contact@example.com
+       wiki_url: https://github.com/searx/searx/wiki
+       twitter_url: https://twitter.com/Searx_engine
 
 ``debug`` :
   Allow a more detailed log if you run searx directly. Display *detailed* error
   messages in the browser too, so this must be deactivated in production.
 
+``contact_url``:
+  Contact ``mailto:`` address or WEB form.
+
+``git_url`` and ``git_branch``:
+  Changes this, to point to your searx fork (branch).
+
+``docs_url``
+  If you host your own documentation, change this URL.
+
+``wiki_url``:
+  Link to your wiki (or ``False``)
+
+``twitter_url``:
+  Link to your tweets (or ``False``)
+
+
+``server:``
+-----------
+
 .. code:: yaml
 
    server:
@@ -90,6 +121,8 @@ Global Settings
 ``default_http_headers``:
   Set additional HTTP headers, see `#755 <https://github.com/searx/searx/issues/715>`__
 
+``outgoing:``
+-------------
 
 .. code:: yaml
 
@@ -139,6 +172,10 @@ Global Settings
   If you use multiple network interfaces, define from which IP the requests must
   be made. This parameter is ignored when ``proxies`` is set.
 
+
+``locales:``
+------------
+
 .. code:: yaml
 
    locales:
@@ -244,13 +281,21 @@ Engine settings
 use_default_settings
 ====================
 
-.. note::
+.. sidebar:: ``use_default_settings: True``
 
-   If searx is cloned from a git repository, most probably there is no need to have an user settings.
+   - :ref:`settings location`
+   - :ref:`use_default_settings.yml`
+   - :origin:`/etc/searx/settings.yml <utils/templates/etc/searx/use_default_settings.yml>`
 
-The user defined settings.yml can relied on the default configuration :origin:`searx/settings.yml` using ``use_default_settings: True``.
+The user defined ``settings.yml`` is loaded from the :ref:`settings location`
+and can relied on the default configuration :origin:`searx/settings.yml` using:
 
-In the following example, the actual settings are the default settings defined in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and the ``bind_address``:
+ ``use_default_settings: True``
+
+``server:``
+  In the following example, the actual settings are the default settings defined
+  in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and
+  the ``bind_address``:
 
   .. code-block:: yaml
 
@@ -259,9 +304,11 @@ In the following example, the actual settings are the default settings defined i
         secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
         bind_address: "0.0.0.0"
 
-With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
+``engines:``
-
+  With ``use_default_settings: True``, each settings can be override in a
-In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
+  similar way, the ``engines`` section is merged according to the engine
+  ``name``.  In this example, searx will load all the engine and the arch linux
+  wiki engine has a :ref:`token<private engines>`:
 
   .. code-block:: yaml
 
@@ -272,7 +319,10 @@ In this example, searx will load all the engine and the arch linux wiki engine h
       - name: arch linux wiki
         tokens: ['$ecretValue']
 
-It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
+``engines:`` / ``remove:``
+  It is possible to remove some engines from the default settings. The following
+  example is similar to the above one, but searx doesn't load the the google
+  engine:
 
   .. code-block:: yaml
 
@@ -286,7 +336,9 @@ It is possible to remove some engines from the default settings. The following e
       - name: arch linux wiki
         tokens: ['$ecretValue']
 
-As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
+``engines:`` / ``keep_only:``
+  As an alternative, it is possible to specify the engines to keep. In the
+  following example, searx has only two engines:
 
   .. code-block:: yaml
 

docs/blog/lxcdev-202006.rst

@@ -259,8 +259,8 @@ suite.  For this, we have to keep an eye on the :ref:`installation basic`:
 - virtualenv in: ``/usr/local/searx/searx-pyenv``
 - searx software in: ``/usr/local/searx/searx-src``
 
-The searx software is a clone of the ``GIT_URL`` (see :ref:`makefile setup`) and
+The searx software is a clone of the ``git_url`` (see :ref:`settings global`) and
-the working tree is checked out from the ``GIT_BRANCH``.  With the use of the
+the working tree is checked out from the ``git_branch``.  With the use of the
 :ref:`searx.sh` the searx service was installed as :ref:`uWSGI application
 <searx uwsgi>`.  To maintain this service, we can use ``systemctl`` (compare
 :ref:`service architectures on distributions <uwsgi configuration>`).

docs/conf.py

@@ -2,14 +2,10 @@
 
 import  sys, os
 from sphinx_build_tools import load_sphinx_config
-from searx.version import VERSION_STRING
 from pallets_sphinx_themes import ProjectLink
 
-from searx.brand import GIT_URL
+from searx import brand
-GIT_BRANCH = os.environ.get("GIT_BRANCH", "master")
+from searx.version import VERSION_STRING
-from searx.brand import SEARX_URL
-from searx.brand import DOCS_URL
-
 
 # Project --------------------------------------------------------------
 
@@ -46,10 +42,10 @@ extlinks['wiki'] = ('https://github.com/searx/searx/wiki/%s', ' ')
 extlinks['pull'] = ('https://github.com/searx/searx/pull/%s', 'PR ')
 
 # links to custom brand
-extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://')
+extlinks['origin'] = (brand.GIT_URL + '/blob/' + brand.GIT_BRANCH + '/%s', 'git://')
-extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
+extlinks['patch'] = (brand.GIT_URL + '/commit/%s', '#')
-extlinks['search'] = (SEARX_URL + '/%s', '#')
+extlinks['search'] = (brand.SEARX_URL + '/%s', '#')
-extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
+extlinks['docs'] = (brand.DOCS_URL + '/%s', 'docs: ')
 extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ')
 extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
 #extlinks['role'] = (
@@ -104,14 +100,20 @@ imgmath_font_size = 14
 # sphinx.ext.imgmath setup END
 
 html_theme_options = {"index_sidebar_logo": True}
-html_context = {
+html_context = {"project_links": [] }
-    "project_links": [
+if brand.GIT_URL:
-        ProjectLink("Source", GIT_URL),
+    html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))
-        ProjectLink("Wiki", "https://github.com/searx/searx/wiki"),
+if brand.WIKI_URL:
-        ProjectLink("Public instances", "https://searx.space/"),
+    html_context["project_links"].append(ProjectLink("Wiki", brand.WIKI_URL))
-        ProjectLink("Twitter", "https://twitter.com/Searx_engine"),
+if brand.PUBLIC_INSTANCES:
-    ]
+    html_context["project_links"].append(ProjectLink("Public instances", brand.PUBLIC_INSTANCES))
-}
+if brand.TWITTER_URL:
+    html_context["project_links"].append(ProjectLink("Twitter", brand.TWITTER_URL))
+if brand.ISSUE_URL:
+    html_context["project_links"].append(ProjectLink("Issue Tracker", brand.ISSUE_URL))
+if brand.CONTACT_URL:
+    html_context["project_links"].append(ProjectLink("Contact", brand.CONTACT_URL))
+
 html_sidebars = {
     "**": ["project.html", "relations.html", "searchbox.html"],
 }

docs/dev/makefile.rst

@@ -8,8 +8,7 @@ Makefile Targets
 
 .. sidebar:: build environment
 
-   Before looking deeper at the targets, first read about :ref:`makefile setup`
+   Before looking deeper at the targets, first read about :ref:`make pyenv`.
-   and :ref:`make pyenv`.
 
    To install system requirements follow :ref:`buildhosts`.
 
@@ -28,37 +27,6 @@ Calling the ``help`` target gives a first overview (``make help``):
    :local:
    :backlinks: entry
 
-
-.. _makefile setup:
-
-Makefile setup
-==============
-
-.. _git stash: https://git-scm.com/docs/git-stash
-
-.. sidebar:: fork & upstream
-
-   Commit changes in your (local) branch, fork or whatever, but do not push them
-   upstream / `git stash`_ is your friend.
-
-The main setup is done in the :origin:`Makefile`.
-
-.. literalinclude:: ../../Makefile
-   :start-after: START Makefile setup
-   :end-before: END Makefile setup
-
-:GIT_URL:    Changes this, to point to your searx fork.
-:GIT_BRANCH: Changes this, to point to your searx branch.
-:SEARX_URL:  Changes this, to point to your searx instance.
-:DOCS_URL:   If you host your own (*brand*) documentation, change this URL.
-
-If you change any of this build environment variables, you have to run ``make
-buildenv``::
-
-  $ make buildenv
-  build searx/brand.py
-  build utils/brand.env
-
 .. _make pyenv:
 
 Python environment
@@ -148,7 +116,7 @@ clean`` stop all processes using :ref:`make pyenv`.
 We describe the usage of the ``doc*`` targets in the :ref:`How to contribute /
 Documentation <contrib docs>` section.  If you want to edit the documentation
 read our :ref:`make docs-live` section.  If you are working in your own brand,
-adjust your :ref:`Makefile setup <makefile setup>`.
+adjust your :ref:`settings global`.
 
 .. _make books:
 
@@ -185,8 +153,8 @@ Use ``make docs-help`` to see which books available:
 ``make gh-pages``
 =================
 
-To deploy on github.io first adjust your :ref:`Makefile setup <makefile
+To deploy on github.io first adjust your :ref:`settings global`.  For any
-setup>`.  For any further read :ref:`deploy on github.io`.
+further read :ref:`deploy on github.io`.
 
 .. _make test:
 

docs/utils/index.rst

@@ -47,8 +47,8 @@ Scripts to maintain services often dispose of common commands and environments.
 Tooling box setup
 =================
 
-The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile
+The main setup is done in the :origin:`.config.sh` (read also :ref:`settings
-setup`).
+global`).
 
 .. literalinclude:: ../../.config.sh
    :language: bash

manage.sh

@@ -123,9 +123,9 @@ docker_build() {
     SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); }  print}')
 
     # add the suffix "-dirty" if the repository has uncommited change
-    # /!\ HACK for searx/searx: ignore searx/brand.py and utils/brand.env
+    # /!\ HACK for searx/searx: ignore utils/brand.env
     git update-index -q --refresh
-    if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'searx/brand.py' | grep -v 'utils/brand.env')" ]; then
+    if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
 	SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
     fi
 

searx/__init__.py

@@ -60,3 +60,49 @@ if 'SEARX_SECRET' in environ:
     settings['server']['secret_key'] = environ['SEARX_SECRET']
 if 'SEARX_BIND_ADDRESS' in environ:
     settings['server']['bind_address'] = environ['SEARX_BIND_ADDRESS']
+
+
+class _brand_namespace:
+
+    @classmethod
+    def get_val(cls, group, name, default=''):
+        return settings.get(group, {}).get(name) or default
+
+    @property
+    def SEARX_URL(self):
+        return self.get_val('server', 'base_url')
+
+    @property
+    def CONTACT_URL(self):
+        return self.get_val('general', 'contact_url')
+
+    @property
+    def GIT_URL(self):
+        return self.get_val('brand', 'git_url')
+
+    @property
+    def GIT_BRANCH(self):
+        return self.get_val('brand', 'git_branch')
+
+    @property
+    def ISSUE_URL(self):
+        return self.get_val('brand', 'issue_url')
+
+    @property
+    def DOCS_URL(self):
+        return self.get_val('brand', 'docs_url')
+
+    @property
+    def PUBLIC_INSTANCES(self):
+        return self.get_val('brand', 'public_instances')
+
+    @property
+    def WIKI_URL(self):
+        return self.get_val('brand', 'wiki_url')
+
+    @property
+    def TWITTER_URL(self):
+        return self.get_val('brand', 'twitter_url')
+
+
+brand = _brand_namespace()

searx/brand.py

@@ -1,7 +0,0 @@
-GIT_URL = 'https://github.com/searx/searx'
-GIT_BRANCH = 'master'
-ISSUE_URL = 'https://github.com/searx/searx/issues'
-SEARX_URL = 'https://searx.me'
-DOCS_URL = 'https://searx.github.io/searx'
-PUBLIC_INSTANCES = 'https://searx.space'
-CONTACT_URL = ''

searx/settings.yml

@@ -1,6 +1,16 @@
 general:
     debug : False # Debug mode, only for development
     instance_name : "searx" # displayed name
+    contact_url: False # mailto:contact@example.com
+
+brand:
+    git_url: https://github.com/searx/searx
+    git_branch: master
+    issue_url: https://github.com/searx/searx/issues
+    docs_url: https://searx.github.io/searx
+    public_instances: https://searx.space
+    wiki_url: https://github.com/searx/searx/wiki
+    twitter_url: https://twitter.com/Searx_engine
 
 search:
     safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict

searx/webapp.py

@@ -1071,6 +1071,7 @@ def config():
         'default_theme': settings['ui']['default_theme'],
         'version': VERSION_STRING,
         'brand': {
+            'CONTACT_URL': brand.CONTACT_URL,
             'GIT_URL': brand.GIT_URL,
             'DOCS_URL': brand.DOCS_URL
         },

setup.py

@@ -7,10 +7,8 @@ from setuptools import find_packages
 import os
 import sys
 
-# required to load VERSION_STRING constant
+from searx.version import VERSION_STRING
-sys.path.insert(0, './searx')
+from searx import brand
-from version import VERSION_STRING
-import brand
 
 with open('README.rst', encoding='utf-8') as f:
     long_description = f.read()

utils/brand.env

@@ -1,7 +1,9 @@
+export SEARX_URL=''
 export GIT_URL='https://github.com/searx/searx'
 export GIT_BRANCH='master'
 export ISSUE_URL='https://github.com/searx/searx/issues'
-export SEARX_URL='https://searx.me'
 export DOCS_URL='https://searx.github.io/searx'
 export PUBLIC_INSTANCES='https://searx.space'
 export CONTACT_URL=''
+export WIKI_URL='https://github.com/searx/searx/wiki'
+export TWITTER_URL='https://twitter.com/Searx_engine'

utils/build_env.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""build environment used by shell scripts
+"""
+
+# set path
+import sys
+import os
+from os.path import realpath, dirname, join, sep, abspath
+
+repo_root = realpath(dirname(realpath(__file__)) + sep + '..')
+sys.path.insert(0, repo_root)
+os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + '/settings.yml')
+
+# Under the assumption that a brand is always a fork assure that the settings
+# file from reposetorie's working tree is used to generate the build_env, not
+# from /etc/searx/settings.yml.
+os.environ['SEARX_SETTINGS_PATH'] = abspath(dirname(__file__) + sep + 'settings.yml')
+
+from searx import brand
+
+name_val = [
+    ('SEARX_URL'              , brand.SEARX_URL),
+    ('GIT_URL'                , brand.GIT_URL),
+    ('GIT_BRANCH'             , brand.GIT_BRANCH),
+    ('ISSUE_URL'              , brand.ISSUE_URL),
+    ('DOCS_URL'               , brand.DOCS_URL),
+    ('PUBLIC_INSTANCES'       , brand.PUBLIC_INSTANCES),
+    ('CONTACT_URL'            , brand.CONTACT_URL),
+    ('WIKI_URL'               , brand.WIKI_URL),
+    ('TWITTER_URL'            , brand.TWITTER_URL),
+]
+
+brand_env = 'utils' + sep + 'brand.env'
+
+print('build %s' % brand_env)
+with open(repo_root + sep + brand_env, 'w', encoding='utf-8') as f:
+    for name, val in name_val:
+        print("export %s='%s'" % (name, val), file=f)