From e17d7632d0ed8284c218f65604ab5fa5d4b8ff81 Mon Sep 17 00:00:00 2001 From: Brock Vojkovic Date: Sun, 11 Aug 2024 08:39:46 +0000 Subject: [PATCH] [feat] add favicons to result urls --- docs/admin/settings/settings_search.rst | 9 ++ searx/favicon_resolver.py | 105 ++++++++++++++++++ searx/preferences.py | 7 +- searx/settings.yml | 3 + searx/settings_defaults.py | 1 + .../themes/simple/img/empty_favicon.svg | 5 + .../static/themes/simple/src/less/search.less | 9 ++ .../themes/simple/src/less/style-ltr.less | 4 + .../themes/simple/src/less/style-rtl.less | 4 + .../static/themes/simple/src/less/style.less | 1 + searx/templates/simple/macros.html | 20 ++++ searx/templates/simple/preferences.html | 3 + .../templates/simple/preferences/favicon.html | 17 +++ searx/webapp.py | 59 ++++++++++ tests/unit/settings/user_settings.yml | 1 + 15 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 searx/favicon_resolver.py create mode 100644 searx/static/themes/simple/img/empty_favicon.svg create mode 100644 searx/templates/simple/preferences/favicon.html diff --git a/docs/admin/settings/settings_search.rst b/docs/admin/settings/settings_search.rst index 836207614..eb63ab684 100644 --- a/docs/admin/settings/settings_search.rst +++ b/docs/admin/settings/settings_search.rst @@ -9,6 +9,7 @@ search: safe_search: 0 autocomplete: "" + favicon_resolver: "" default_lang: "" ban_time_on_fail: 5 max_ban_time_on_fail: 120 @@ -41,6 +42,14 @@ - ``qwant`` - ``wikipedia`` +``favicon_resolver``: + Favicon resolver, leave blank to turn off the feature by default. + + - ``allesedv`` + - ``duckduckgo`` + - ``google`` + - ``yandex`` + ``default_lang``: Default search language - leave blank to detect from browser information or use codes from :origin:`searx/languages.py`. diff --git a/searx/favicon_resolver.py b/searx/favicon_resolver.py new file mode 100644 index 000000000..d292d4ce7 --- /dev/null +++ b/searx/favicon_resolver.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""This module implements functions needed for the favicon resolver. + +""" +# pylint: disable=use-dict-literal + +from httpx import HTTPError + +from searx import settings + +from searx.network import get as http_get, post as http_post +from searx.exceptions import SearxEngineResponseException + + +def update_kwargs(**kwargs): + if 'timeout' not in kwargs: + kwargs['timeout'] = settings['outgoing']['request_timeout'] + kwargs['raise_for_httperror'] = False + + +def get(*args, **kwargs): + update_kwargs(**kwargs) + return http_get(*args, **kwargs) + + +def post(*args, **kwargs): + update_kwargs(**kwargs) + return http_post(*args, **kwargs) + + +def allesedv(domain): + """Favicon Resolver from allesedv.com""" + + url = 'https://f1.allesedv.com/32/{domain}' + + # will just return a 200 regardless of the favicon existing or not + # sometimes will be correct size, sometimes not + response = get(url.format(domain=domain)) + + # returns image/gif if the favicon does not exist + if response.headers['Content-Type'] == 'image/gif': + return [] + + return response.content + + +def duckduckgo(domain): + """Favicon Resolver from duckduckgo.com""" + + url = 'https://icons.duckduckgo.com/ip2/{domain}.ico' + + # will return a 404 if the favicon does not exist and a 200 if it does, + response = get(url.format(domain=domain)) + + # api will respond with a 32x32 png image + if response.status_code == 200: + return response.content + return [] + + +def google(domain): + """Favicon Resolver from google.com""" + + url = 'https://www.google.com/s2/favicons?sz=32&domain={domain}' + + # will return a 404 if the favicon does not exist and a 200 if it does, + response = get(url.format(domain=domain)) + + # api will respond with a 32x32 png image + if response.status_code == 200: + return response.content + return [] + + +def yandex(domain): + """Favicon Resolver from yandex.com""" + + url = 'https://favicon.yandex.net/favicon/{domain}' + + # will always return 200 + response = get(url.format(domain=domain)) + + # api will respond with a 16x16 png image, if it doesn't exist, it will be a 1x1 png image (70 bytes) + if response.status_code == 200: + if len(response.content) > 70: + return response.content + return [] + + +backends = { + 'allesedv': allesedv, + 'duckduckgo': duckduckgo, + 'google': google, + 'yandex': yandex, +} + + +def search_favicon(backend_name, domain): + backend = backends.get(backend_name) + if backend is None: + return [] + try: + return backend(domain) + except (HTTPError, SearxEngineResponseException): + return [] diff --git a/searx/preferences.py b/searx/preferences.py index 4b7494ac2..92758efa6 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -13,7 +13,7 @@ from collections import OrderedDict import flask import babel -from searx import settings, autocomplete +from searx import settings, autocomplete, favicon_resolver from searx.enginelib import Engine from searx.plugins import Plugin from searx.locales import LOCALE_NAMES @@ -406,6 +406,11 @@ class Preferences: locked=is_locked('autocomplete'), choices=list(autocomplete.backends.keys()) + [''] ), + 'favicon_resolver': EnumStringSetting( + settings['search']['favicon_resolver'], + locked=is_locked('favicon_resolver'), + choices=list(favicon_resolver.backends.keys()) + [''] + ), 'image_proxy': BooleanSetting( settings['server']['image_proxy'], locked=is_locked('image_proxy') diff --git a/searx/settings.yml b/searx/settings.yml index 8b264eaf6..5143e69c0 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -35,6 +35,9 @@ search: autocomplete: "" # minimun characters to type before autocompleter starts autocomplete_min: 4 + # backend for the favicon near URL in search results. + # Available resolvers: "allesedv", "duckduckgo", "google", "yandex" - leave blank to turn it off by default. + favicon_resolver: "" # Default search language - leave blank to detect from browser information or # use codes from 'languages.py' default_lang: "auto" diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 7daf4cc20..06cae34bc 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -156,6 +156,7 @@ SCHEMA = { 'safe_search': SettingsValue((0, 1, 2), 0), 'autocomplete': SettingsValue(str, ''), 'autocomplete_min': SettingsValue(int, 4), + 'favicon_resolver': SettingsValue(str, ''), 'default_lang': SettingsValue(tuple(SXNG_LOCALE_TAGS + ['']), ''), 'languages': SettingSublistValue(SXNG_LOCALE_TAGS, SXNG_LOCALE_TAGS), 'ban_time_on_fail': SettingsValue(numbers.Real, 5), diff --git a/searx/static/themes/simple/img/empty_favicon.svg b/searx/static/themes/simple/img/empty_favicon.svg new file mode 100644 index 000000000..f4e3e334d --- /dev/null +++ b/searx/static/themes/simple/img/empty_favicon.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/searx/static/themes/simple/src/less/search.less b/searx/static/themes/simple/src/less/search.less index f18a7ba2c..0c8fbe9f3 100644 --- a/searx/static/themes/simple/src/less/search.less +++ b/searx/static/themes/simple/src/less/search.less @@ -378,3 +378,12 @@ html.no-js #clear_search.hide_if_nojs { #categories_container { position: relative; } + +.favicon img { + height: 1.8rem; + width: 1.8rem; + border-radius: 20%; + background-color: #ddd; + border: 1px solid #ccc; + display: flex; +} diff --git a/searx/static/themes/simple/src/less/style-ltr.less b/searx/static/themes/simple/src/less/style-ltr.less index 6f7218b02..5d8c5dbe5 100644 --- a/searx/static/themes/simple/src/less/style-ltr.less +++ b/searx/static/themes/simple/src/less/style-ltr.less @@ -82,4 +82,8 @@ transform: scale(1, 1); } +.favicon { + margin: 0 8px 0 0; +} + @import "style.less"; diff --git a/searx/static/themes/simple/src/less/style-rtl.less b/searx/static/themes/simple/src/less/style-rtl.less index aa97e039c..aa663436f 100644 --- a/searx/static/themes/simple/src/less/style-rtl.less +++ b/searx/static/themes/simple/src/less/style-rtl.less @@ -96,6 +96,10 @@ .result .url_wrapper { justify-content: end; + + .favicon { + margin: 0 0 0 8px; + } } } diff --git a/searx/static/themes/simple/src/less/style.less b/searx/static/themes/simple/src/less/style.less index d35dd744c..29ae4039e 100644 --- a/searx/static/themes/simple/src/less/style.less +++ b/searx/static/themes/simple/src/less/style.less @@ -234,6 +234,7 @@ article[data-vim-selected].category-social { .url_wrapper { display: flex; + align-items: center; font-size: 1rem; color: var(--color-result-url-font); flex-wrap: nowrap; diff --git a/searx/templates/simple/macros.html b/searx/templates/simple/macros.html index f7af553b6..418f85227 100644 --- a/searx/templates/simple/macros.html +++ b/searx/templates/simple/macros.html @@ -21,9 +21,29 @@ {% macro result_header(result, favicons, image_proxify) -%}
{{- result_open_link(result.url, "url_wrapper") -}} + {% if not rtl %} + {%- if favicon_resolver != "" %} +
+ {{ result.parsed_url.netloc }} +
+ {%- endif -%} + {%- endif -%} {%- for part in get_pretty_url(result.parsed_url) -%} {{- part -}} {%- endfor %} + {% if rtl %} + {%- if favicon_resolver != "" %} +
+ {{ result.parsed_url.netloc }} +
+ {%- endif -%} + {%- endif -%} {{- result_close_link() -}} {%- if result.thumbnail %}{{ result_open_link(result.url) }}{{ result_close_link() }}{% endif -%}

{{ result_link(result.url, result.title|safe) }}

diff --git a/searx/templates/simple/preferences.html b/searx/templates/simple/preferences.html index 825a98fe2..bc96e1198 100644 --- a/searx/templates/simple/preferences.html +++ b/searx/templates/simple/preferences.html @@ -173,6 +173,9 @@ {%- if 'autocomplete' not in locked_preferences -%} {%- include 'simple/preferences/autocomplete.html' -%} {%- endif -%} + {%- if 'favicon' not in locked_preferences -%} + {%- include 'simple/preferences/favicon.html' -%} + {%- endif -%} {% if 'safesearch' not in locked_preferences %} {%- include 'simple/preferences/safesearch.html' -%} {%- endif -%} diff --git a/searx/templates/simple/preferences/favicon.html b/searx/templates/simple/preferences/favicon.html new file mode 100644 index 000000000..207bf2a24 --- /dev/null +++ b/searx/templates/simple/preferences/favicon.html @@ -0,0 +1,17 @@ +
{{- '' -}} + {{- _('Favicon Resolver') -}}{{- '' -}} +
{{- '' -}} + {{- '' -}} +
{{- '' -}} +
+ {{- _('Display favicons near search results') -}} +
{{- '' -}} +
{{- '' -}} diff --git a/searx/webapp.py b/searx/webapp.py index dd79defcb..8046dc392 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -123,6 +123,7 @@ from searx.locales import ( # renaming names from searx imports ... from searx.autocomplete import search_autocomplete, backends as autocomplete_backends +from searx.favicon_resolver import search_favicon, backends as favicon_backends from searx.redisdb import initialize as redis_initialize from searx.sxng_locales import sxng_locales from searx.search import SearchWithPlugins, initialize as search_initialize @@ -297,6 +298,24 @@ def morty_proxify(url: str): return '{0}?{1}'.format(settings['result_proxy']['url'], urlencode(url_params)) +def favicon_proxify(url: str): + # url is a FQDN (e.g. example.com, en.wikipedia.org) + + resolver = request.preferences.get_value('favicon_resolver') + + # if resolver is empty, just return nothing + if not resolver: + return "" + + # check resolver is valid + if resolver not in favicon_backends: + return "" + + h = new_hmac(settings['server']['secret_key'], url.encode()) + + return '{0}?{1}'.format(url_for('favicon_proxy'), urlencode(dict(q=url.encode(), h=h))) + + def image_proxify(url: str): if url.startswith('//'): @@ -358,6 +377,7 @@ def get_client_settings(): return { 'autocomplete_provider': req_pref.get_value('autocomplete'), 'autocomplete_min': get_setting('search.autocomplete_min'), + 'favicon_resolver': req_pref.get_value('favicon_resolver'), 'http_method': req_pref.get_value('method'), 'infinite_scroll': req_pref.get_value('infinite_scroll'), 'translations': get_translations(), @@ -388,6 +408,7 @@ def render(template_name: str, **kwargs): # values from the preferences kwargs['preferences'] = request.preferences kwargs['autocomplete'] = request.preferences.get_value('autocomplete') + kwargs['favicon_resolver'] = request.preferences.get_value('favicon_resolver') kwargs['infinite_scroll'] = request.preferences.get_value('infinite_scroll') kwargs['search_on_category_select'] = request.preferences.get_value('search_on_category_select') kwargs['hotkeys'] = request.preferences.get_value('hotkeys') @@ -431,6 +452,7 @@ def render(template_name: str, **kwargs): # helpers to create links to other pages kwargs['url_for'] = custom_url_for # override url_for function in templates kwargs['image_proxify'] = image_proxify + kwargs['favicon_proxify'] = favicon_proxify kwargs['proxify'] = morty_proxify if settings['result_proxy']['url'] is not None else None kwargs['proxify_results'] = settings['result_proxy']['proxify_results'] kwargs['cache_url'] = settings['ui']['cache_url'] @@ -873,6 +895,42 @@ def autocompleter(): return Response(suggestions, mimetype=mimetype) +@app.route('/favicon', methods=['GET']) +def favicon_proxy(): + """Return proxied favicon results""" + url = request.args.get('q') + + # malformed request + if not url: + return '', 400 + + # malformed request / does not have authorisation + if not is_hmac_of(settings['server']['secret_key'], url.encode(), request.args.get('h', '')): + return '', 400 + + resolver = request.preferences.get_value('favicon_resolver') + + # check if the favicon resolver is valid + if not resolver or resolver not in favicon_backends: + return '', 400 + + # parse query + raw_text_query = RawTextQuery(url, []) + + resp = search_favicon(resolver, raw_text_query) + + # return 404 if the favicon is not found + if not resp: + theme = request.preferences.get_value("theme") + # return favicon from /static/themes/simple/img/empty_favicon.svg + # we can't rely on an onerror event in the img tag to display a default favicon as this violates the CSP. + # using redirect to save network bandwidth (user will have this location cached). + return redirect(url_for('static', filename='themes/' + theme + '/img/empty_favicon.svg')) + + # will always return a PNG image + return Response(resp, mimetype='image/png') + + @app.route('/preferences', methods=['GET', 'POST']) def preferences(): """Render preferences page && save user preferences""" @@ -1020,6 +1078,7 @@ def preferences(): ], disabled_engines = disabled_engines, autocomplete_backends = autocomplete_backends, + favicon_backends = favicon_backends, shortcuts = {y: x for x, y in engine_shortcuts.items()}, themes = themes, plugins = plugins, diff --git a/tests/unit/settings/user_settings.yml b/tests/unit/settings/user_settings.yml index c4c4d74ef..ed82c450a 100644 --- a/tests/unit/settings/user_settings.yml +++ b/tests/unit/settings/user_settings.yml @@ -5,6 +5,7 @@ general: search: safe_search: 0 autocomplete: "" + favicon_resolver: "" default_lang: "" ban_time_on_fail: 5 max_ban_time_on_fail: 120