Merge pull request #930 from return42/merge-user-doc2

Integrate the user documentation into the application

docs/_themes/searxng/static/searxng.css

@@ -14,7 +14,7 @@ p.version-warning {
   background-color: #004b6b;
 }
 
-div.sidebar {
+aside.sidebar {
   background-color: whitesmoke;
   border-color: lightsteelblue;
   border-radius: 3pt;

docs/conf.py

@@ -35,7 +35,7 @@ master_doc = "index"
 source_suffix = '.rst'
 numfig = True
 
-exclude_patterns = ['build-templates/*.rst']
+exclude_patterns = ['build-templates/*.rst', 'user/*.md']
 
 import searx.engines
 import searx.plugins
@@ -94,7 +94,6 @@ extlinks['pull-searx'] = ('https://github.com/searx/searx/pull/%s', 'PR ')
 # links to custom brand
 extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://')
 extlinks['patch'] = (GIT_URL + '/commit/%s', '#')
-extlinks['search'] = (SEARXNG_URL + '/%s', '#')
 extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ')
 extlinks['pypi'] = ('https://pypi.org/project/%s', 'PyPi: ')
 extlinks['man'] = ('https://manpages.debian.org/jump?q=%s', '')
@@ -117,14 +116,17 @@ extensions = [
     "sphinx.ext.intersphinx",
     "pallets_sphinx_themes",
     "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst
-    "sphinxcontrib.jinja",  # https://github.com/tardyp/sphinx-jinja
+    "sphinx_jinja",  # https://github.com/tardyp/sphinx-jinja
     "sphinxcontrib.programoutput",  # https://github.com/NextThought/sphinxcontrib-programoutput
     'linuxdoc.kernel_include',  # Implementation of the 'kernel-include' reST-directive.
     'linuxdoc.rstFlatTable',    # Implementation of the 'flat-table' reST-directive.
     'linuxdoc.kfigure',         # Sphinx extension which implements scalable image handling.
     "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs
+    'myst_parser',  # https://www.sphinx-doc.org/en/master/usage/markdown.html
 ]
 
+suppress_warnings = ['myst.domains']
+
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3/", None),
     "flask": ("https://flask.palletsprojects.com/", None),

docs/index.rst

@@ -31,6 +31,7 @@ If you don't trust anyone, you can set up your own, see :ref:`installation`.
    :caption: Contents
 
    user/index
+   own-instance
    admin/index
    dev/index
    utils/index

docs/src/searx.infopage.rst

@@ -0,0 +1,8 @@
+.. _searx.infopage:
+
+================
+Online ``/info``
+================
+
+.. automodule:: searx.infopage
+  :members:

docs/user/.gitignore

@@ -0,0 +1 @@
+*.md

docs/user/index.rst

@@ -2,9 +2,14 @@
 User documentation
 ==================
 
-.. toctree::
+.. contents:: Contents
-   :maxdepth: 2
+   :depth: 3
-   :caption: Contents
+   :local:
+   :backlinks: entry
+
+
+.. _search-syntax:
+
+.. include:: search-syntax.md
+   :parser: myst_parser.sphinx_
 
-   search_syntax
-   own-instance

docs/user/search_syntax.rst

@@ -1,39 +0,0 @@
-
-.. _search-syntax:
-
-=============
-Search syntax
-=============
-
-SearXNG allows you to modify the default categories, engines and search language
-via the search query.
-
-Prefix ``!``
-  to set Category/engine
-
-Prefix: ``:``
-  to set language
-
-Abbrevations of the engines and languages are also accepted.  Engine/category
-modifiers are chainable and inclusive (e.g. with :search:`!it !ddg !wp qwer
-<?q=%21it%20%21ddg%20%21wp%20qwer>` search in IT category **and** duckduckgo
-**and** wikipedia for ``qwer``).
-
-See the :search:`/preferences page <preferences>` for the list of engines,
-categories and languages.
-
-Examples
-========
-
-Search in wikipedia for ``qwer``:
-
-- :search:`!wp qwer <?q=%21wp%20qwer>` or
-- :search:`!wikipedia qwer :search:<?q=%21wikipedia%20qwer>`
-
-Image search:
-
-- :search:`!images Cthulhu <?q=%21images%20Cthulhu>`
-
-Custom language in wikipedia:
-
-- :search:`:hu !wp hackerspace <?q=%3Ahu%20%21wp%20hackerspace>`

manage

@@ -419,6 +419,7 @@ docs.prebuild() {
         ./utils/searx.sh doc   | cat > "${DOCS_BUILD}/includes/searx.rst"
         ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
         ./utils/morty.sh doc   | cat > "${DOCS_BUILD}/includes/morty.rst"
+        pyenv.cmd searxng_extra/docs_prebuild
     )
     dump_return $?
 }

requirements-dev.txt

@@ -10,10 +10,11 @@ twine==3.8.0
 Pallets-Sphinx-Themes==2.0.2
 Sphinx==4.4.0
 sphinx-issues==3.0.1
-sphinx-jinja==1.4.0
+sphinx-jinja==2.0.1
-sphinx-tabs==3.2.0
+sphinx-tabs @ git+https://github.com/return42/sphinx-tabs.git@fix-152#egg=fix-152
 sphinxcontrib-programoutput==0.17
 sphinx-autobuild==2021.3.14
+myst-parser==0.17.0
 linuxdoc==20211220
 aiounittest==1.4.1
 yamllint==1.26.3

searx/compat.py

@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+# pyright: basic
+"""Module for backward compatibility.
+
+"""
+# pylint: disable=C,R
+
+
+try:
+    from functools import cached_property  # pylint: disable=unused-import
+
+except ImportError:
+
+    # cache_property has been added in py3.8 [1]
+    #
+    # To support cache_property in py3.7 the implementation from 3.8 has been
+    # copied here.  This code can be cleanup with EOL of py3.7.
+    #
+    # [1] https://docs.python.org/3/library/functools.html#functools.cached_property
+
+    from threading import RLock
+
+    _NOT_FOUND = object()
+
+    class cached_property:
+        def __init__(self, func):
+            self.func = func
+            self.attrname = None
+            self.__doc__ = func.__doc__
+            self.lock = RLock()
+
+        def __set_name__(self, owner, name):
+            if self.attrname is None:
+                self.attrname = name
+            elif name != self.attrname:
+                raise TypeError(
+                    "Cannot assign the same cached_property to two different names "
+                    f"({self.attrname!r} and {name!r})."
+                )
+
+        def __get__(self, instance, owner=None):
+            if instance is None:
+                return self
+            if self.attrname is None:
+                raise TypeError("Cannot use cached_property instance without calling __set_name__ on it.")
+            try:
+                cache = instance.__dict__
+            except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
+                msg = (
+                    f"No '__dict__' attribute on {type(instance).__name__!r} "
+                    f"instance to cache {self.attrname!r} property."
+                )
+                raise TypeError(msg) from None
+            val = cache.get(self.attrname, _NOT_FOUND)
+            if val is _NOT_FOUND:
+                with self.lock:
+                    # check if another thread filled cache while we awaited lock
+                    val = cache.get(self.attrname, _NOT_FOUND)
+                    if val is _NOT_FOUND:
+                        val = self.func(instance)
+                        try:
+                            cache[self.attrname] = val
+                        except TypeError:
+                            msg = (
+                                f"The '__dict__' attribute on {type(instance).__name__!r} instance "
+                                f"does not support item assignment for caching {self.attrname!r} property."
+                            )
+                            raise TypeError(msg) from None
+            return val

searx/infopage/__init__.py

@@ -0,0 +1,179 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+# pyright: basic
+"""Render SearXNG instance documentation.
+
+Usage in a Flask app route:
+
+.. code:: python
+
+  from searx import infopage
+
+  _INFO_PAGES = infopage.InfoPageSet(infopage.MistletoePage)
+
+  @app.route('/info/<pagename>', methods=['GET'])
+  def info(pagename):
+
+      locale = request.preferences.get_value('locale')
+      page = _INFO_PAGES.get_page(pagename, locale)
+
+"""
+
+__all__ = ['InfoPage', 'InfoPageSet']
+
+import os
+import os.path
+import logging
+import typing
+
+import urllib.parse
+import jinja2
+from flask.helpers import url_for
+import mistletoe
+
+from .. import get_setting
+from ..compat import cached_property
+from ..version import GIT_URL
+from ..locales import LOCALE_NAMES
+
+
+logger = logging.getLogger('searx.infopage')
+_INFO_FOLDER = os.path.abspath(os.path.dirname(__file__))
+
+
+class InfoPage:
+    """A page of the :py:obj:`online documentation <InfoPageSet>`."""
+
+    def __init__(self, fname):
+        self.fname = fname
+
+    @cached_property
+    def raw_content(self):
+        """Raw content of the page (without any jinja rendering)"""
+        with open(self.fname, 'r', encoding='utf-8') as f:
+            return f.read()
+
+    @cached_property
+    def content(self):
+        """Content of the page (rendered in a Jinja conntext)"""
+        ctx = self.get_ctx()
+        template = jinja2.Environment().from_string(self.raw_content)
+        return template.render(**ctx)
+
+    @cached_property
+    def title(self):
+        """Title of the content (without any markup)"""
+        t = ""
+        for l in self.raw_content.split('\n'):
+            if l.startswith('# '):
+                t = l.strip('# ')
+        return t
+
+    @cached_property
+    def html(self):
+        """Render Markdown (CommonMark_) to HTML by using mistletoe_.
+
+        .. _CommonMark: https://commonmark.org/
+        .. _mistletoe: https://github.com/miyuchina/mistletoe
+
+        """
+        return mistletoe.markdown(self.content)
+
+    def get_ctx(self):  # pylint: disable=no-self-use
+        """Jinja context to render :py:obj:`InfoPage.content`"""
+
+        def _md_link(name, url):
+            url = url_for(url, _external=True)
+            return "[%s](%s)" % (name, url)
+
+        def _md_search(query):
+            url = '%s?q=%s' % (url_for('search', _external=True), urllib.parse.quote(query))
+            return '[%s](%s)' % (query, url)
+
+        ctx = {}
+        ctx['GIT_URL'] = GIT_URL
+        ctx['get_setting'] = get_setting
+        ctx['link'] = _md_link
+        ctx['search'] = _md_search
+
+        return ctx
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} fname={self.fname!r}>'
+
+
+class InfoPageSet:  # pylint: disable=too-few-public-methods
+    """Cached rendering of the online documentation a SearXNG instance has.
+
+    :param page_class: render online documentation by :py:obj:`InfoPage` parser.
+    :type page_class: :py:obj:`InfoPage`
+
+    :param info_folder: information directory
+    :type info_folder: str
+    """
+
+    def __init__(
+        self, page_class: typing.Optional[typing.Type[InfoPage]] = None, info_folder: typing.Optional[str] = None
+    ):
+        self.page_class = page_class or InfoPage
+        self.CACHE: typing.Dict[tuple, typing.Optional[InfoPage]] = {}
+
+        # future: could be set from settings.xml
+
+        self.folder: str = info_folder or _INFO_FOLDER
+        """location of the Markdwon files"""
+
+        self.locale_default: str = 'en'
+        """default language"""
+
+        self.locales: typing.List = [locale for locale in os.listdir(_INFO_FOLDER) if locale in LOCALE_NAMES]
+        """list of supported languages (aka locales)"""
+
+        self.toc: typing.List = [
+            'search-syntax',
+            'about',
+        ]
+        """list of articles in the online documentation"""
+
+    def get_page(self, pagename: str, locale: typing.Optional[str] = None):
+        """Return ``pagename`` instance of :py:obj:`InfoPage`
+
+        :param pagename: name of the page, a value from :py:obj:`InfoPageSet.toc`
+        :type pagename: str
+
+        :param locale: language of the page, e.g. ``en``, ``zh_Hans_CN``
+                       (default: :py:obj:`InfoPageSet.i18n_origin`)
+        :type locale: str
+
+        """
+        locale = locale or self.locale_default
+
+        if pagename not in self.toc:
+            return None
+        if locale not in self.locales:
+            return None
+
+        cache_key = (pagename, locale)
+        page = self.CACHE.get(cache_key)
+
+        if page is not None:
+            return page
+
+        # not yet instantiated
+
+        fname = os.path.join(self.folder, locale, pagename) + '.md'
+        if not os.path.exists(fname):
+            logger.info('file %s does not exists', fname)
+            self.CACHE[cache_key] = None
+            return None
+
+        page = self.page_class(fname)
+        self.CACHE[cache_key] = page
+        return page
+
+    def all_pages(self, locale: typing.Optional[str] = None):
+        """Iterate over all pages of the TOC"""
+        locale = locale or self.locale_default
+        for pagename in self.toc:
+            page = self.get_page(pagename, locale)
+            yield pagename, page

searx/infopage/en/about.md

@@ -1,30 +1,29 @@
 # About SearXNG
 
 SearXNG is a fork from the well-known [searx] [metasearch engine], aggregating
-the results of other [search engines][url_for:preferences] while not storing
+the results of other {{link('search engines', 'preferences')}} while not
-information about its users.
+storing information about its users.
 
 More about SearXNG ...
 
-* [SearXNG sources][brand.git_url]
+* [SearXNG sources]({{GIT_URL}})
 * [weblate]
 
----
 
 ## Why use it?
 
-* SearXNG may not offer you as personalised results as Google,
+* SearXNG may not offer you as personalised results as Google, but it doesn't
-  but it doesn't generate a profile about you.
+  generate a profile about you.
 
-* SearXNG doesn't care about what you search for, never shares anything
+* SearXNG doesn't care about what you search for, never shares anything with a
-  with a third party, and it can't be used to compromise you.
+  third party, and it can't be used to compromise you.
 
-* SearXNG is free software, the code is 100% open and you can help
+* SearXNG is free software, the code is 100% open and you can help to make it
-  to make it better.  See more on [SearXNG sources][brand.git_url].
+  better.  See more on [SearXNG sources]({{GIT_URL}}).
 
-If you do care about privacy, want to be a conscious user, or otherwise
+If you do care about privacy, want to be a conscious user, or otherwise believe
-believe in digital freedom, make SearXNG your default search engine or run
+in digital freedom, make SearXNG your default search engine or run it on your
-it on your own server
+own server
 
 ## Technical details - How does it work?
 
@@ -37,35 +36,40 @@ exception: searx uses the search bar to perform GET requests.  SearXNG can be
 added to your browser's search bar; moreover, it can be set as the default
 search engine.
 
-<span id='add to browser'></span>
 ## How to set as the default search engine?
 
 SearXNG supports [OpenSearch].  For more information on changing your default
 search engine, see your browser's documentation:
 
-* [Firefox](https://support.mozilla.org/en-US/kb/add-or-remove-search-engine-firefox)
+* [Firefox]
-* [Microsoft Edge](https://support.microsoft.com/en-us/help/4028574/microsoft-edge-change-the-default-search-engine)
+* [Microsoft Edge]
-* Chromium-based browsers [only add websites that the user navigates to without a path.](https://www.chromium.org/tab-to-search)
+* Chromium-based browsers [only add websites that the user navigates to without
+  a path.](https://www.chromium.org/tab-to-search)
 
 ## Where to find anonymous usage statistics of this instance ?
 
-[Stats page][url_for:stats] contains some useful data about the engines used.
+{{link('Stats page', 'stats')}} contains some useful data about the engines
+used.
 
 ## How can I make it my own?
 
-SearXNG appreciates your concern regarding logs, so take the code from
+SearXNG appreciates your concern regarding logs, so take the code from the
-the [SearXNG project][brand.git_url] and run it yourself!
+[SearXNG project]({{GIT_URL}}) and run it yourself!
 
-Add your instance to this [list of public instances][brand.public_instances] to
+Add your instance to this [list of public
-help other people reclaim their privacy and make the Internet freer!  The more
+instances]({{get_setting('brand.public_instances')}}) to help other people
-decentralized the Internet is, the more freedom we have!
+reclaim their privacy and make the Internet freer!  The more decentralized the
+Internet is, the more freedom we have!
 
 ## Where are the docs & code of this instance?
 
-See the [SearXNG docs][brand.docs_url] and [SearXNG sources][brand.git_url]
+See the [SearXNG docs]({{get_setting('brand.docs_url')}}) and [SearXNG
+sources]({{GIT_URL}})
 
 [searx]: https://github.com/searx/searx
 [metasearch engine]: https://en.wikipedia.org/wiki/Metasearch_engine
 [weblate]: https://weblate.bubu1.eu/projects/searxng/
 [seeks project]: https://beniz.github.io/seeks/
 [OpenSearch]: https://github.com/dewitt/opensearch/blob/master/opensearch-1-1-draft-6.md
+[Firefox]: https://support.mozilla.org/en-US/kb/add-or-remove-search-engine-firefox
+[Microsoft Edge]: https://support.microsoft.com/en-us/help/4028574/microsoft-edge-change-the-default-search-engine

searx/infopage/en/search-syntax.md

@@ -0,0 +1,35 @@
+# Search syntax
+
+SearXNG allows you to modify the default categories, engines and search language
+via the search query.
+
+Prefix `!` to set category and engine names.
+
+Prefix: `:` to set the language.
+
+Abbrevations of the engines and languages are also accepted.  Engine/category
+modifiers are chainable and inclusive.  E.g. with {{search('!map !ddg !wp paris')}}
+search in map category **and** duckduckgo **and** wikipedia for
+`paris`.
+
+See the {{link('preferences', 'preferences')}} for the list of engines,
+categories and languages.
+
+## Examples
+
+Search in wikipedia for `paris`:
+
+* {{search('!wp paris')}}
+* {{search('!wikipedia paris')}}
+
+Search in category `map` for `paris`:
+
+* {{search('!map paris')}}
+
+Image search:
+
+* {{search('!images Wau Holland')}}
+
+Custom language in wikipedia:
+
+* {{search(':fr !wp Wau Holland')}}

searx/static/themes/oscar/js/searxng.min.js

@@ -4,7 +4,7 @@
  * (C) Copyright Contributors to the searx project (2014 - 2021).
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
-window.searxng=function(t){"use strict";t.getElementsByTagName("html")[0].className="js";var e=t.currentScript||(e=t.getElementsByTagName("script"))[e.length-1];return{autocompleter:"true"===e.getAttribute("data-autocompleter"),infinite_scroll:"true"===e.getAttribute("data-infinite-scroll"),method:e.getAttribute("data-method"),translations:JSON.parse(e.getAttribute("data-translations"))}}(document),
+window.searxng=function(t){"use strict";t.getElementsByTagName("html")[0].className="js";t=t.currentScript||(t=t.getElementsByTagName("script"))[t.length-1];return{autocompleter:"true"===t.getAttribute("data-autocompleter"),infinite_scroll:"true"===t.getAttribute("data-infinite-scroll"),method:t.getAttribute("data-method"),translations:JSON.parse(t.getAttribute("data-translations"))}}(document),
 /**
  * @license
  * (C) Copyright Contributors to the SearXNG project.
@@ -20,7 +20,7 @@ $(document).ready(function(){var t,n="";searxng.autocompleter&&((t=new Bloodhoun
  * (C) 2014 by Thomas Pointhuber, <thomas.pointhuber@gmx.at>
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
-$(document).ready(function(){$("#q.autofocus").focus(),$("#clear_search").click(function(){document.getElementById("q").value=""}),$(".select-all-on-click").click(function(){$(this).select()}),$(".btn-collapse").click(function(){var t=$(this).data("btn-text-collapsed"),e=$(this).data("btn-text-not-collapsed");""!==t&&""!==e&&(new_html=$(this).hasClass("collapsed")?$(this).html().replace(t,e):$(this).html().replace(e,t),$(this).html(new_html))}),$(".btn-toggle .btn").click(function(){var t="btn-"+$(this).data("btn-class"),e=$(this).data("btn-label-default"),n=$(this).data("btn-label-toggled");""!==n&&(new_html=$(this).hasClass("btn-default")?$(this).html().replace(e,n):$(this).html().replace(n,e),$(this).html(new_html)),$(this).toggleClass(t),$(this).toggleClass("btn-default")}),$(".media-loader").click(function(){var t=$(this).data("target"),e=$(t+" > iframe"),t=e.attr("src");void 0!==t&&!1!==t||e.attr("src",e.data("src"))}),$(".btn-sm").dblclick(function(){var t="btn-"+$(this).data("btn-class");$(this).hasClass("btn-default")?($(".btn-sm > input").attr("checked","checked"),$(".btn-sm > input").prop("checked",!0),$(".btn-sm").addClass(t),$(".btn-sm").addClass("active"),$(".btn-sm").removeClass("btn-default")):($(".btn-sm > input").attr("checked",""),$(".btn-sm > input").removeAttr("checked"),$(".btn-sm > input").checked=!1,$(".btn-sm").removeClass(t),$(".btn-sm").removeClass("active"),$(".btn-sm").addClass("btn-default"))}),$(".nav-tabs").click(function(t){$(t.target).parents("ul").children().attr("aria-selected","false"),$(t.target).parent().attr("aria-selected","true")}),searxng.image_thumbnail_layout=new searxng.ImageLayout("#main_results","#main_results .result-images","img.img-thumbnail",15,3,200),searxng.image_thumbnail_layout.watch()}),
+$(document).ready(function(){$("#q.autofocus").focus(),$("#clear_search").click(function(){document.getElementById("q").value=""}),$(".select-all-on-click").click(function(){$(this).select()}),$(".btn-collapse").click(function(){var t=$(this).data("btn-text-collapsed"),e=$(this).data("btn-text-not-collapsed");""!==t&&""!==e&&(new_html=$(this).hasClass("collapsed")?$(this).html().replace(t,e):$(this).html().replace(e,t),$(this).html(new_html))}),$(".btn-toggle .btn").click(function(){var t="btn-"+$(this).data("btn-class"),e=$(this).data("btn-label-default"),n=$(this).data("btn-label-toggled");""!==n&&(new_html=$(this).hasClass("btn-default")?$(this).html().replace(e,n):$(this).html().replace(n,e),$(this).html(new_html)),$(this).toggleClass(t),$(this).toggleClass("btn-default")}),$(".media-loader").click(function(){var t=$(this).data("target"),t=$(t+" > iframe"),e=t.attr("src");void 0!==e&&!1!==e||t.attr("src",t.data("src"))}),$(".btn-sm").dblclick(function(){var t="btn-"+$(this).data("btn-class");$(this).hasClass("btn-default")?($(".btn-sm > input").attr("checked","checked"),$(".btn-sm > input").prop("checked",!0),$(".btn-sm").addClass(t),$(".btn-sm").addClass("active"),$(".btn-sm").removeClass("btn-default")):($(".btn-sm > input").attr("checked",""),$(".btn-sm > input").removeAttr("checked"),$(".btn-sm > input").checked=!1,$(".btn-sm").removeClass(t),$(".btn-sm").removeClass("active"),$(".btn-sm").addClass("btn-default"))}),$(".nav-tabs").click(function(t){$(t.target).parents("ul").children().attr("aria-selected","false"),$(t.target).parent().attr("aria-selected","true")}),searxng.image_thumbnail_layout=new searxng.ImageLayout("#main_results","#main_results .result-images","img.img-thumbnail",15,3,200),searxng.image_thumbnail_layout.watch()}),
 /**
 *
 * Google Image Layout v0.0.1
@@ -42,7 +42,7 @@ $(document).ready(function(){$("#q.autofocus").focus(),$("#clear_search").click(
 * );
 * searxng.image_thumbnail_layout.watch();
 */
-function(s,c){function t(t,e,n,a,i,o){this.container_selector=t,this.results_selector=e,this.img_selector=n,this.verticalMargin=a,this.horizontalMargin=i,this.maxHeight=o,this.trottleCallToAlign=null,this.alignAfterThrotteling=!1}t.prototype._getHeigth=function(t,e){for(var n,a=0,i=0;i<t.length;i++)0<(n=t[i]).naturalWidth&&0<n.naturalHeight?a+=n.naturalWidth/n.naturalHeight:a+=1;return(e-t.length*this.verticalMargin)/a},t.prototype._setSize=function(t,e){for(var n,a,i=t.length,o=0;o<i;o++)n=0<(a=t[o]).naturalWidth&&0<a.naturalHeight?e*a.naturalWidth/a.naturalHeight:e,a.setAttribute("width",Math.round(n)),a.setAttribute("height",Math.round(e)),a.style.marginLeft=Math.round(this.horizontalMargin)+"px",a.style.marginTop=Math.round(this.horizontalMargin)+"px",a.style.marginRight=Math.round(this.verticalMargin-7)+"px",a.style.marginBottom=Math.round(this.verticalMargin-7)+"px",(a=a.parentNode.parentNode).classList.contains("js")||a.classList.add("js")},t.prototype._alignImgs=function(t){for(var e,n,a,i,o=c.querySelector(this.container_selector),s=window.getComputedStyle(o),r=parseInt(s.getPropertyValue("padding-left"),10),s=parseInt(s.getPropertyValue("padding-right"),10),l=o.clientWidth-r-s;0<t.length;){for(e=!0,a=1;a<=t.length&&e;a++)n=t.slice(0,a),(i=this._getHeigth(n,l))<this.maxHeight&&(this._setSize(n,i),t=t.slice(a),e=!1);if(e){this._setSize(n,Math.min(this.maxHeight,i));break}}},t.prototype.throttleAlign=function(){var t=this;t.trottleCallToAlign?t.alignAfterThrotteling=!0:(t.alignAfterThrotteling=!1,t.align(),t.trottleCallToAlign=setTimeout(function(){t.alignAfterThrotteling&&t.align(),t.alignAfterThrotteling=!1,t.trottleCallToAlign=null},20))},t.prototype.align=function(){for(var t=c.querySelectorAll(this.results_selector),e=t.length,n=null,a=null,i=[],o=0;o<e;o++)(a=t[o]).previousElementSibling!==n&&0<i.length&&(this._alignImgs(i),i=[]),i.push(a.querySelector(this.img_selector)),n=a;0<i.length&&this._alignImgs(i)},t.prototype._monitorImages=function(){var t,e,n=this.throttleAlign.bind(this),a=c.querySelectorAll(this.results_selector),i=a.length;function o(t){t.originalTarget.src=s.searxng.static_path+s.searxng.theme.img_load_error}for(t=0;t<i;t++)null==(e=a[t].querySelector(this.img_selector))||e.classList.contains("aligned")||(e.addEventListener("load",n),e.addEventListener("error",n),e.addEventListener("timeout",n),s.searxng.theme.img_load_error&&e.addEventListener("error",o,{once:!0}),e.classList.add("aligned"))},t.prototype.watch=function(){var t=this.throttleAlign.bind(this);s.addEventListener("pageshow",t),s.addEventListener("load",t),s.addEventListener("resize",t),this._monitorImages();var a=this;let e=new MutationObserver(e=>{let n=!1;for(let t=0;t<e.length;t++)if(0<e[t].addedNodes.length&&e[t].addedNodes[0].classList.contains("result")){n=!0;break}n&&a._monitorImages()});e.observe(c.querySelector(this.container_selector),{childList:!0,subtree:!0,attributes:!1,characterData:!1})},s.searxng.ImageLayout=t}(window,document),
+function(s,c){function t(t,e,n,a,i,o){this.container_selector=t,this.results_selector=e,this.img_selector=n,this.verticalMargin=a,this.horizontalMargin=i,this.maxHeight=o,this.trottleCallToAlign=null,this.alignAfterThrotteling=!1}t.prototype._getHeigth=function(t,e){for(var n,a=0,i=0;i<t.length;i++)0<(n=t[i]).naturalWidth&&0<n.naturalHeight?a+=n.naturalWidth/n.naturalHeight:a+=1;return(e-t.length*this.verticalMargin)/a},t.prototype._setSize=function(t,e){for(var n,a,i=t.length,o=0;o<i;o++)a=0<(n=t[o]).naturalWidth&&0<n.naturalHeight?e*n.naturalWidth/n.naturalHeight:e,n.setAttribute("width",Math.round(a)),n.setAttribute("height",Math.round(e)),n.style.marginLeft=Math.round(this.horizontalMargin)+"px",n.style.marginTop=Math.round(this.horizontalMargin)+"px",n.style.marginRight=Math.round(this.verticalMargin-7)+"px",n.style.marginBottom=Math.round(this.verticalMargin-7)+"px",(a=n.parentNode.parentNode).classList.contains("js")||a.classList.add("js")},t.prototype._alignImgs=function(t){for(var e,n,a,i,o=c.querySelector(this.container_selector),s=window.getComputedStyle(o),r=parseInt(s.getPropertyValue("padding-left"),10),s=parseInt(s.getPropertyValue("padding-right"),10),l=o.clientWidth-r-s;0<t.length;){for(e=!0,a=1;a<=t.length&&e;a++)n=t.slice(0,a),(i=this._getHeigth(n,l))<this.maxHeight&&(this._setSize(n,i),t=t.slice(a),e=!1);if(e){this._setSize(n,Math.min(this.maxHeight,i));break}}},t.prototype.throttleAlign=function(){var t=this;t.trottleCallToAlign?t.alignAfterThrotteling=!0:(t.alignAfterThrotteling=!1,t.align(),t.trottleCallToAlign=setTimeout(function(){t.alignAfterThrotteling&&t.align(),t.alignAfterThrotteling=!1,t.trottleCallToAlign=null},20))},t.prototype.align=function(){for(var t=c.querySelectorAll(this.results_selector),e=t.length,n=null,a=null,i=[],o=0;o<e;o++)(a=t[o]).previousElementSibling!==n&&0<i.length&&(this._alignImgs(i),i=[]),i.push(a.querySelector(this.img_selector)),n=a;0<i.length&&this._alignImgs(i)},t.prototype._monitorImages=function(){var t,e,n=this.throttleAlign.bind(this),a=c.querySelectorAll(this.results_selector),i=a.length;function o(t){t.originalTarget.src=s.searxng.static_path+s.searxng.theme.img_load_error}for(t=0;t<i;t++)null==(e=a[t].querySelector(this.img_selector))||e.classList.contains("aligned")||(e.addEventListener("load",n),e.addEventListener("error",n),e.addEventListener("timeout",n),s.searxng.theme.img_load_error&&e.addEventListener("error",o,{once:!0}),e.classList.add("aligned"))},t.prototype.watch=function(){var t=this.throttleAlign.bind(this),a=(s.addEventListener("pageshow",t),s.addEventListener("load",t),s.addEventListener("resize",t),this._monitorImages(),this);let e=new MutationObserver(e=>{let n=!1;for(let t=0;t<e.length;t++)if(0<e[t].addedNodes.length&&e[t].addedNodes[0].classList.contains("result")){n=!0;break}n&&a._monitorImages()});e.observe(c.querySelector(this.container_selector),{childList:!0,subtree:!0,attributes:!1,characterData:!1})},s.searxng.ImageLayout=t}(window,document),
 /**
  * @license
  * (C) Copyright Contributors to the SearXNG project.
@@ -64,7 +64,7 @@ window.addEventListener("load",function(){$(".infobox").each(function(){var t=$(
  * (C) 2014 by Thomas Pointhuber, <thomas.pointhuber@gmx.at>
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
-$(document).ready(function(){$(".searxng_init_map").on("click",function(t){var e=$(this).data("leaflet-target"),n=$(this).data("map-lon"),a=$(this).data("map-lat"),i=$(this).data("map-zoom"),o=$(this).data("map-boundingbox"),s=$(this).data("map-geojson");o&&(southWest=L.latLng(o[0],o[2]),northEast=L.latLng(o[1],o[3]),map_bounds=L.latLngBounds(southWest,northEast)),L.Icon.Default.imagePath="./static/themes/oscar/css/images/";var r=L.map(e),e=new L.TileLayer("https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",{minZoom:1,maxZoom:19,attribution:'Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors'});new L.TileLayer("https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png",{minZoom:1,maxZoom:19,attribution:'Wikimedia maps beta | Maps data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors'});setTimeout(function(){map_bounds?r.fitBounds(map_bounds,{maxZoom:17}):n&&a&&(i?r.setView(new L.LatLng(a,n),i):r.setView(new L.LatLng(a,n),8))},0),r.addLayer(e),L.control.layers({"OSM Mapnik":e}).addTo(r),s&&L.geoJson(s).addTo(r),$(this).off(t)})}),
+$(document).ready(function(){$(".searxng_init_map").on("click",function(t){var e=$(this).data("leaflet-target"),n=$(this).data("map-lon"),a=$(this).data("map-lat"),i=$(this).data("map-zoom"),o=$(this).data("map-boundingbox"),s=$(this).data("map-geojson"),r=(o&&(southWest=L.latLng(o[0],o[2]),northEast=L.latLng(o[1],o[3]),map_bounds=L.latLngBounds(southWest,northEast)),L.Icon.Default.imagePath="./static/themes/oscar/css/images/",L.map(e)),o=new L.TileLayer("https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",{minZoom:1,maxZoom:19,attribution:'Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors'}),e=(new L.TileLayer("https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png",{minZoom:1,maxZoom:19,attribution:'Wikimedia maps beta | Maps data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors'}),setTimeout(function(){map_bounds?r.fitBounds(map_bounds,{maxZoom:17}):n&&a&&(i?r.setView(new L.LatLng(a,n),i):r.setView(new L.LatLng(a,n),8))},0),r.addLayer(o),{"OSM Mapnik":o});L.control.layers(e).addTo(r),s&&L.geoJson(s).addTo(r),$(this).off(t)})}),
 /**
  * @license
  * (C) Copyright Contributors to the SearXNG project.

searx/static/themes/simple/src/less/definitions.less

@@ -109,6 +109,8 @@
   --color-toolkit-engine-tooltip-background: #fff;
   --color-toolkit-loader-border: rgba(0, 0, 0, 0.2);
   --color-toolkit-loader-borderleft: rgba(255, 255, 255, 0);
+  --color-doc-code: #300;
+  --color-doc-code-background: #fdd;
 }
 
 .dark-themes() {
@@ -215,6 +217,8 @@
   --color-toolkit-engine-tooltip-background: #222;
   --color-toolkit-loader-border: rgba(255, 255, 255, 0.2);
   --color-toolkit-loader-borderleft: rgba(0, 0, 0, 0);
+  --color-doc-code: #fdd;
+  --color-doc-code-background: #300;
 }
 
 /// Dark Theme (autoswitch based on device pref)

searx/static/themes/simple/src/less/info.less

@@ -0,0 +1,13 @@
+.info-page {
+  font-family: sans-serif;
+  font-size: 1.3em;
+
+  code {
+    font-family: monospace;
+    font-size: 1.3em;
+    color: var(--color-doc-code);
+    background-color: var(--color-doc-code-background);
+    padding: 2px 5px;
+    .rounded-corners(5px);
+  }
+}

searx/static/themes/simple/src/less/style.less

@@ -18,6 +18,7 @@
 @import "detail.less";
 @import "animations.less";
 @import "embedded.less";
+@import "info.less";
 
 // for index.html template
 @import "index.less";

searx/templates/oscar/help.html

@@ -1,12 +0,0 @@
-{% extends "oscar/base.html" %}
-{% block title %}{{ page.title }} - {% endblock %}
-{% block content %}
-<ul class="nav nav-tabs">
-{% for name, page in all_pages %}
-  <li {% if name == page_filename %}class="active"{% endif %}>
-    <a href="{{name}}">{{page.title}}</a>
-  </li>
-{% endfor %}
-</ul>
-{{ page.content | safe }}
-{% endblock %}

searx/templates/oscar/info.html

@@ -0,0 +1,12 @@
+{% extends "oscar/base.html" %}
+{% block title %}{{ active_page.title }} - {% endblock %}
+{% block content %}
+<ul class="nav nav-tabs">
+  {% for pagename, page, locale in all_pages %}
+  <li>
+    <a href="{{ url_for('info', pagename=pagename, locale=locale) }}" {% if pagename == active_pagename %}class="active"{% endif %}>{{page.title}}</a>
+  </li>
+{% endfor %}
+</ul>
+{{ active_page.html | safe }}
+{% endblock %}

searx/templates/oscar/navbar.html

@@ -3,7 +3,7 @@
         <a href="{{ url_for('index') }}">{{ instance_name }}</a>{{- "" -}}
     </span>{{- "" -}}
     <span class="{% if rtl %}pull-left{% else %}pull-right{% endif %}">{{- "" -}}
-        <a href="{{ url_for('help_page', pagename='about') }}">{{ _('about') }}</a>{{- "" -}}
+        <a href="{{ url_for('info', pagename='about') }}">{{ _('about') }}</a>{{- "" -}}
         <a href="{{ url_for('preferences') }}">{{ _('preferences') }}</a>{{- "" -}}
     </span>{{- "" -}}
 </div>

searx/templates/simple/base.html

@@ -58,7 +58,7 @@
   </main>
   <footer>
     <p>
-    {{ _('Powered by') }} <a href="{{ url_for('help_page', pagename='about') }}">searxng</a> - {{ searx_version }} — {{ _('a privacy-respecting, hackable metasearch engine') }}<br/>
+    {{ _('Powered by') }} <a href="{{ url_for('info', pagename='about') }}">searxng</a> - {{ searx_version }} — {{ _('a privacy-respecting, hackable metasearch engine') }}<br/>
         <a href="{{ searx_git_url }}">{{ _('Source code') }}</a> |
         <a href="{{ get_setting('brand.issue_url') }}">{{ _('Issue tracker') }}</a> |
         <a href="{{ url_for('stats') }}">{{ _('Engine stats') }}</a> |

searx/templates/simple/help.html

@@ -1,12 +0,0 @@
-{% extends 'simple/page_with_header.html' %}
-{% block title %}{{ page.title }} - {% endblock %}
-{% block content %}
-<ul class="tabs">
-{% for name, page in all_pages %}
-  <li>
-    <a href="{{name}}" {% if name == page_filename %}class="active"{% endif %}>{{page.title}}</a>
-  </li>
-{% endfor %}
-</ul>
-{{ page.content | safe }}
-{% endblock %}

searx/templates/simple/info.html

@@ -0,0 +1,14 @@
+{% extends 'simple/page_with_header.html' %}
+{% block title %}{{ active_page.title }} - {% endblock %}
+{% block content %}
+<ul class="tabs">
+{% for pagename, page, locale in all_pages %}
+  <li>
+    <a href="{{ url_for('info', pagename=pagename, locale=locale) }}" {% if pagename == active_pagename %}class="active"{% endif %}>{{page.title}}</a>
+  </li>
+{% endfor %}
+</ul>
+<div class="info-page {{pagename}}">
+  {{- active_page.html | safe -}}
+</div>
+{% endblock %}

searx/user_help.py

@@ -1,61 +0,0 @@
-# pyright: basic
-from typing import Dict, NamedTuple
-import pkg_resources
-
-import flask
-from flask.helpers import url_for
-import mistletoe
-
-from . import get_setting
-from .version import GIT_URL
-
-
-class HelpPage(NamedTuple):
-    title: str
-    content: str
-
-
-# Whenever a new .md file is added to help/ it needs to be added here
-_TOC = ('about',)
-
-PAGES: Dict[str, HelpPage] = {}
-""" Maps a filename under help/ without the file extension to the rendered page. """
-
-
-def render(app: flask.Flask):
-    """
-    Renders the user documentation. Must be called after all Flask routes have been
-    registered, because the documentation might try to link to them with Flask's `url_for`.
-
-    We render the user documentation once on startup to improve performance.
-    """
-
-    link_targets = {
-        'brand.git_url': GIT_URL,
-        'brand.public_instances': get_setting('brand.public_instances'),
-        'brand.docs_url': get_setting('brand.docs_url'),
-    }
-
-    base_url = get_setting('server.base_url') or None
-    # we specify base_url so that url_for works for base_urls that have a non-root path
-
-    with app.test_request_context(base_url=base_url):
-        link_targets['url_for:index'] = url_for('index')
-        link_targets['url_for:preferences'] = url_for('preferences')
-        link_targets['url_for:stats'] = url_for('stats')
-
-    define_link_targets = ''.join(f'[{name}]: {url}\n' for name, url in link_targets.items())
-
-    for pagename in _TOC:
-        file_content = pkg_resources.resource_string(__name__, 'help/' + pagename + '.md').decode()
-        markdown = define_link_targets + file_content
-        assert file_content.startswith('# ')
-        title = file_content.split('\n', maxsplit=1)[0].strip('# ')
-        content: str = mistletoe.markdown(markdown)
-
-        if pagename == 'about':
-            try:
-                content += pkg_resources.resource_string(__name__, 'templates/__common__/aboutextend.html').decode()
-            except FileNotFoundError:
-                pass
-        PAGES[pagename] = HelpPage(title=title, content=content)

searx/webapp.py

@@ -56,8 +56,9 @@ from searx import (
     get_setting,
     settings,
     searx_debug,
-    user_help,
 )
+
+from searx import infopage
 from searx.data import ENGINE_DESCRIPTIONS
 from searx.results import Timing, UnresponsiveEngine
 from searx.settings_defaults import OUTPUT_FORMATS
@@ -382,6 +383,11 @@ def url_for_theme(endpoint: str, override_theme: Optional[str] = None, **values)
         if file_hash:
             values['filename'] = filename_with_theme
             suffix = "?" + file_hash
+    if endpoint == 'info' and 'locale' not in values:
+        locale = request.preferences.get_value('locale')
+        if _INFO_PAGES.get_page(values['pagename'], locale) is None:
+            locale = _INFO_PAGES.locale_default
+        values['locale'] = locale
     return url_for(endpoint, **values) + suffix
 
 
@@ -660,6 +666,7 @@ def index():
         # fmt: off
         'index.html',
         selected_categories=get_selected_categories(request.preferences, request.form),
+        current_locale = request.preferences.get_value("locale"),
         # fmt: on
     )
 
@@ -864,6 +871,7 @@ def search():
         unresponsive_engines = __get_translated_errors(
             result_container.unresponsive_engines
         ),
+        current_locale = request.preferences.get_value("locale"),
         current_language = match_language(
             search_query.lang,
             settings['search']['languages'],
@@ -898,19 +906,36 @@ def __get_translated_errors(unresponsive_engines: Iterable[UnresponsiveEngine]):
 @app.route('/about', methods=['GET'])
 def about():
     """Redirect to about page"""
-    return redirect(url_for('help_page', pagename='about'))
+    locale = request.preferences.get_value('locale')
+    return redirect(url_for('info', pagename='about', locale=locale))
 
 
-@app.route('/help/en/<pagename>', methods=['GET'])
+_INFO_PAGES = infopage.InfoPageSet()
-def help_page(pagename):
-    """Render help page"""
-    page = user_help.PAGES.get(pagename)
 
+
+@app.route('/info/<locale>/<pagename>', methods=['GET'])
+def info(pagename, locale):
+    """Render page of online user documentation"""
+
+    page = _INFO_PAGES.get_page(pagename, locale)
     if page is None:
         flask.abort(404)
 
+    def all_pages():
+        user_locale = request.preferences.get_value('locale')
+        for for_pagename, for_page in _INFO_PAGES.all_pages(user_locale):
+            for_locale = locale
+            if for_page is None:
+                # we are sure that for_pagename != pagename
+                for_page = _INFO_PAGES.get_page(for_pagename, _INFO_PAGES.locale_default)
+                for_locale = _INFO_PAGES.locale_default
+            yield for_pagename, for_page, for_locale
+
     return render(
-        'help.html', page=user_help.PAGES[pagename], all_pages=user_help.PAGES.items(), page_filename=pagename
+        'info.html',
+        all_pages=all_pages(),
+        active_page=page,
+        active_pagename=pagename,
     )
 
 
@@ -1411,7 +1436,6 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai
 if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
     plugin_initialize(app)
     search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
-    user_help.render(app)
 
 
 def run():

searxng_extra/docs_prebuild

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# lint: pylint
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""Script that implements some prebuild tasks needed by target docs.prebuild
+"""
+
+import sys
+import os.path
+import time
+from contextlib import contextmanager
+from searx import settings, get_setting
+from searx.infopage import InfoPageSet, InfoPage
+
+
+_doc_user = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'docs', 'user'))
+
+
+def main():
+    base_url = get_setting('server.base_url', None)
+    if base_url:
+        infopageset_ctx = _instance_infosetset_ctx(base_url)
+    else:
+        infopageset_ctx = _offline_infosetset_ctx()
+
+    with infopageset_ctx as infopageset:
+        for _, page in infopageset.all_pages('en'):
+            fname = os.path.join(_doc_user, os.path.basename(page.fname))
+            with open(fname, 'w') as f:
+                f.write(page.content)
+
+
+class OfflinePage(InfoPage):
+
+    def get_ctx(self):  # pylint: disable=no-self-use
+        """Jinja context to render :py:obj:`DocPage.content` for offline purpose (no
+        links to SearXNG instance)"""
+
+        ctx = super().get_ctx()
+        ctx['link'] = lambda name, url: '`%s`' % name
+        ctx['search'] = lambda query: '`%s`' % query
+
+        return ctx
+
+
+@contextmanager
+def _offline_infosetset_ctx():
+    yield InfoPageSet(OfflinePage)
+
+
+@contextmanager
+def _instance_infosetset_ctx(base_url):
+    # The url_for functions in the jinja templates need all routes to be
+    # registered in the Flask app.
+
+    settings['server']['secret_key'] = ''
+    from searx.webapp import app
+
+    # Specify base_url so that url_for() works for base_urls.  If base_url is
+    # specified, then these values from are given preference over any Flask's
+    # generics (see flaskfix.py).
+
+    with app.test_request_context(base_url=base_url):
+        yield InfoPageSet()
+
+    # The searx.webapp import from above fires some HTTP requests, thats
+    # why we get a RuntimeError::
+    #
+    #     RuntimeError: The connection pool was closed while 1 HTTP \
+    #       requests/responses were still in-flight.
+    #
+    # Closing network won't help ..
+    #   from searx.network import network
+    #   network.done()
+
+    # waiting some seconds before ending the comand line was the only solution I
+    # found ..
+
+    time.sleep(3)
+    return DOC
+
+
+if __name__ == '__main__':
+    sys.exit(main())

setup.py

@@ -58,7 +58,8 @@ setup(
             '../requirements.txt',
             '../requirements-dev.txt',
             'data/*',
-            'help/*',
+            'info/*',
+            'info/*/*',
             'plugins/*/*',
             'static/*.*',
             'static/*/*.*',

tests/unit/test_webapp.py

@@ -177,10 +177,14 @@ class ViewsTestCase(SearxTestCase):
 
         self.assertIn(b'<description>first test content</description>', result.data)
 
-    def test_about(self):
+    def test_redirect_about(self):
-        result = self.app.get('/help/en/about')
+        result = self.app.get('/about')
+        self.assertEqual(result.status_code, 302)
+
+    def test_info_page(self):
+        result = self.app.get('/info/en/search-syntax')
         self.assertEqual(result.status_code, 200)
-        self.assertIn(b'<h1>About SearXNG</h1>', result.data)
+        self.assertIn(b'<h1>Search syntax</h1>', result.data)
 
     def test_health(self):
         result = self.app.get('/healthz')