Merge pull request #882 from return42/fix-873

[fix] replace embedded HTML by data_src

searx/engines/bandcamp.py

@@ -4,7 +4,7 @@ Bandcamp (Music)
 @website     https://bandcamp.com/
 @provide-api no
 @results     HTML
-@parse       url, title, content, publishedDate, embedded, thumbnail
+@parse       url, title, content, publishedDate, iframe_src, thumbnail
 """
 
 from urllib.parse import urlencode, urlparse, parse_qs
@@ -27,10 +27,7 @@ paging = True
 
 base_url = "https://bandcamp.com/"
 search_string = search_string = 'search?{query}&page={page}'
-embedded_url = '''<iframe width="100%" height="166"
-    scrolling="no" frameborder="no"
-    data-src="https://bandcamp.com/EmbeddedPlayer/{type}={result_id}/size=large/bgcol=ffffff/linkcol=0687f5/tracklist=false/artwork=small/transparent=true/"
-></iframe>'''
+iframe_src = "https://bandcamp.com/EmbeddedPlayer/{type}={result_id}/size=large/bgcol=ffffff/linkcol=0687f5/tracklist=false/artwork=small/transparent=true/"
 
 
 def request(query, params):
@@ -74,8 +71,9 @@ def response(resp):
         if thumbnail:
             new_result['thumbnail'] = thumbnail[0]
         if "album" in result.classes:
-            new_result["embedded"] = embedded_url.format(type='album', result_id=result_id)
+            new_result["iframe_src"] = iframe_src.format(type='album', result_id=result_id)
         elif "track" in result.classes:
-            new_result["embedded"] = embedded_url.format(type='track', result_id=result_id)
+            new_result["iframe_src"] = iframe_src.format(type='track', result_id=result_id)
+
         results.append(new_result)
     return results

searx/engines/dailymotion.py

@@ -25,11 +25,6 @@ paging = True
 # search-url
 # see http://www.dailymotion.com/doc/api/obj-video.html
 search_url = 'https://api.dailymotion.com/videos?fields=created_time,title,description,duration,url,thumbnail_360_url,id&sort=relevance&limit=5&page={pageno}&{query}'  # noqa
-embedded_url = (
-    '<iframe frameborder="0" width="540" height="304" '
-    + 'data-src="https://www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>'
-)
-
 supported_languages_url = 'https://api.dailymotion.com/languages'
 
 
@@ -64,7 +59,6 @@ def response(resp):
         content = html_to_text(res['description'])
         thumbnail = res['thumbnail_360_url']
         publishedDate = datetime.fromtimestamp(res['created_time'], None)
-        embedded = embedded_url.format(videoid=res['id'])
 
         # http to https
         thumbnail = thumbnail.replace("http://", "https://")
@@ -76,7 +70,7 @@ def response(resp):
                 'title': title,
                 'content': content,
                 'publishedDate': publishedDate,
-                'embedded': embedded,
+                'iframe_src': "https://www.dailymotion.com/embed/video/" + res['id'],
                 'thumbnail': thumbnail,
             }
         )

searx/engines/deezer.py

@@ -23,13 +23,7 @@ paging = True
 # search-url
 url = 'https://api.deezer.com/'
 search_url = url + 'search?{query}&index={offset}'
-
-embedded_url = (
-    '<iframe scrolling="no" frameborder="0" allowTransparency="true" '
-    + 'data-src="https://www.deezer.com/plugins/player?type=tracks&id={audioid}" '
-    + 'width="540" height="80"></iframe>'
-)
-
+iframe_src = "https://www.deezer.com/plugins/player?type=tracks&id={audioid}"
 
 # do search-request
 def request(query, params):
@@ -57,10 +51,10 @@ def response(resp):
 
             content = '{} - {} - {}'.format(result['artist']['name'], result['album']['title'], result['title'])
 
-            embedded = embedded_url.format(audioid=result['id'])
-
             # append result
-            results.append({'url': url, 'title': title, 'embedded': embedded, 'content': content})
+            results.append(
+                {'url': url, 'title': title, 'iframe_src': iframe_src.format(audioid=result['id']), 'content': content}
+            )
 
     # return results
     return results

searx/engines/freesound.py

@@ -29,9 +29,6 @@ search_url = (
     url + "search/text/?query={query}&page={page}&fields=name,url,download,created,description,type&token={api_key}"
 )
 
-embedded_url = '<audio controls><source src="{uri}" type="audio/{ftype}"></audio>'
-
-
 # search request
 def request(query, params):
     params["url"] = search_url.format(
@@ -52,7 +49,6 @@ def response(resp):
         content = result["description"][:128]
         publishedDate = datetime.fromisoformat(result["created"])
         uri = result["download"]
-        embedded = embedded_url.format(uri=uri, ftype=result["type"])
 
         # append result
         results.append(
@@ -60,7 +56,7 @@ def response(resp):
                 "url": result["url"],
                 "title": title,
                 "publishedDate": publishedDate,
-                "embedded": embedded,
+                "audio_src": uri,
                 "content": content,
             }
         )

searx/engines/invidious.py

@@ -58,14 +58,6 @@ def response(resp):
     results = []
 
     search_results = resp.json()
-    embedded_url = (
-        '<iframe width="540" height="304" '
-        + 'data-src="'
-        + resp.search_params['base_url']
-        + '/embed/{videoid}" '
-        + 'frameborder="0" allowfullscreen></iframe>'
-    )
-
     base_invidious_url = resp.search_params['base_url'] + "/watch?v="
 
     for result in search_results:
@@ -76,7 +68,6 @@ def response(resp):
                 continue
 
             url = base_invidious_url + videoid
-            embedded = embedded_url.format(videoid=videoid)
             thumbs = result.get("videoThumbnails", [])
             thumb = next((th for th in thumbs if th["quality"] == "sddefault"), None)
             if thumb:
@@ -100,7 +91,7 @@ def response(resp):
                     "template": "videos.html",
                     "author": result.get("author"),
                     "publishedDate": publishedDate,
-                    "embedded": embedded,
+                    "iframe_src": resp.search_params['base_url'] + '/embed/' + videoid,
                     "thumbnail": thumbnail,
                 }
             )

searx/engines/mixcloud.py

@@ -24,12 +24,7 @@ paging = True
 # search-url
 url = 'https://api.mixcloud.com/'
 search_url = url + 'search/?{query}&type=cloudcast&limit=10&offset={offset}'
-
-embedded_url = (
-    '<iframe scrolling="no" frameborder="0" allowTransparency="true" '
-    + 'data-src="https://www.mixcloud.com/widget/iframe/?feed={url}" width="300" height="300"></iframe>'
-)
-
+iframe_src = "https://www.mixcloud.com/widget/iframe/?feed={url}"
 
 # do search-request
 def request(query, params):
@@ -51,12 +46,17 @@ def response(resp):
         title = result['name']
         url = result['url']
         content = result['user']['name']
-        embedded = embedded_url.format(url=url)
         publishedDate = parser.parse(result['created_time'])
 
         # append result
         results.append(
-            {'url': url, 'title': title, 'embedded': embedded, 'publishedDate': publishedDate, 'content': content}
+            {
+                'url': url,
+                'title': title,
+                'iframe_src': iframe_src.format(url=url),
+                'publishedDate': publishedDate,
+                'content': content,
+            }
         )
 
     # return results

searx/engines/peertube.py

@@ -51,12 +51,6 @@ def response(resp):
 
     search_res = loads(resp.text)
 
-    embedded_url = (
-        '<iframe width="560" height="315" sandbox="allow-same-origin allow-scripts allow-popups" '
-        + 'src="'
-        + sanitized_url
-        + '{embed_path}" frameborder="0" allowfullscreen></iframe>'
-    )
     # return empty array if there are no results
     if "data" not in search_res:
         return []
@@ -72,7 +66,6 @@ def response(resp):
             content = ""
         thumbnail = sanitized_url + res["thumbnailPath"]
         publishedDate = datetime.strptime(res["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
-        embedded = embedded_url.format(embed_path=res["embedPath"])
 
         results.append(
             {
@@ -81,7 +74,7 @@ def response(resp):
                 "title": title,
                 "content": content,
                 "publishedDate": publishedDate,
-                "embedded": embedded,
+                "iframe_src": sanitized_url + res["embedPath"],
                 "thumbnail": thumbnail,
             }
         )

searx/engines/sepiasearch.py

@@ -41,9 +41,6 @@ time_range_table = {
 }
 
 
-embedded_url = '<iframe width="540" height="304" src="{url}" frameborder="0" allowfullscreen></iframe>'
-
-
 def minute_to_hm(minute):
     if isinstance(minute, int):
         return "%d:%02d" % (divmod(minute, 60))
@@ -88,7 +85,6 @@ def response(resp):
         content = result['description']
         thumbnail = result['thumbnailUrl']
         publishedDate = parser.parse(result['publishedAt'])
-        embedded = embedded_url.format(url=result.get('embedUrl'))
         author = result.get('account', {}).get('displayName')
         length = minute_to_hm(result.get('duration'))
         url = result['url']
@@ -102,7 +98,7 @@ def response(resp):
                 'length': length,
                 'template': 'videos.html',
                 'publishedDate': publishedDate,
-                'embedded': embedded,
+                'iframe_src': result.get('embedUrl'),
                 'thumbnail': thumbnail,
             }
         )

searx/engines/soundcloud.py

@@ -37,12 +37,6 @@ search_url = (
     '&client_id={client_id}'
 )  # noqa
 
-embedded_url = (
-    '<iframe width="100%" height="166" '
-    + 'scrolling="no" frameborder="no" '
-    + 'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'
-)
-
 cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)
 guest_client_id = ''
 
@@ -97,7 +91,6 @@ def response(resp):
             content = result['description'] or ''
             publishedDate = parser.parse(result['last_modified'])
             uri = quote_plus(result['uri'])
-            embedded = embedded_url.format(uri=uri)
 
             # append result
             results.append(
@@ -105,7 +98,7 @@ def response(resp):
                     'url': result['permalink_url'],
                     'title': title,
                     'publishedDate': publishedDate,
-                    'embedded': embedded,
+                    'iframe_src': "https://w.soundcloud.com/player/?url=" + uri,
                     'content': content,
                 }
             )

searx/engines/spotify.py

@@ -29,10 +29,6 @@ api_client_secret = None
 url = 'https://api.spotify.com/'
 search_url = url + 'v1/search?{query}&type=track&offset={offset}'
 
-embedded_url = '<iframe data-src="https://embed.spotify.com/?uri=spotify:track:{audioid}"\
-     width="300" height="80" frameborder="0" allowtransparency="true"></iframe>'
-
-
 # do search-request
 def request(query, params):
     offset = (params['pageno'] - 1) * 20
@@ -66,10 +62,15 @@ def response(resp):
             url = result['external_urls']['spotify']
             content = '{} - {} - {}'.format(result['artists'][0]['name'], result['album']['name'], result['name'])
 
-            embedded = embedded_url.format(audioid=result['id'])
-
             # append result
-            results.append({'url': url, 'title': title, 'embedded': embedded, 'content': content})
+            results.append(
+                {
+                    'url': url,
+                    'title': title,
+                    'iframe_src': "https://embed.spotify.com/?uri=spotify:track:" + result['id'],
+                    'content': content,
+                }
+            )
 
     # return results
     return results

searx/engines/vimeo.py

@@ -25,12 +25,6 @@ paging = True
 base_url = 'https://vimeo.com/'
 search_url = base_url + '/search/page:{pageno}?{query}'
 
-embedded_url = (
-    '<iframe data-src="https://player.vimeo.com/video/{videoid}" '
-    + 'width="540" height="304" frameborder="0" '
-    + 'webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'
-)
-
 
 # do search-request
 def request(query, params):
@@ -54,7 +48,6 @@ def response(resp):
         title = result['name']
         thumbnail = result['pictures']['sizes'][-1]['link']
         publishedDate = parser.parse(result['created_time'])
-        embedded = embedded_url.format(videoid=videoid)
 
         # append result
         results.append(
@@ -64,7 +57,7 @@ def response(resp):
                 'content': '',
                 'template': 'videos.html',
                 'publishedDate': publishedDate,
-                'embedded': embedded,
+                'iframe_src': "https://player.vimeo.com/video/" + videoid,
                 'thumbnail': thumbnail,
             }
         )

searx/engines/youtube_api.py

@@ -26,13 +26,6 @@ api_key = None
 # search-url
 base_url = 'https://www.googleapis.com/youtube/v3/search'
 search_url = base_url + '?part=snippet&{query}&maxResults=20&key={api_key}'
-
-embedded_url = (
-    '<iframe width="540" height="304" '
-    + 'data-src="https://www.youtube-nocookie.com/embed/{videoid}" '
-    + 'frameborder="0" allowfullscreen></iframe>'
-)
-
 base_youtube_url = 'https://www.youtube.com/watch?v='
 
 
@@ -77,8 +70,6 @@ def response(resp):
 
         url = base_youtube_url + videoid
 
-        embedded = embedded_url.format(videoid=videoid)
-
         # append result
         results.append(
             {
@@ -87,7 +78,7 @@ def response(resp):
                 'content': content,
                 'template': 'videos.html',
                 'publishedDate': publishedDate,
-                'embedded': embedded,
+                'iframe_src': "https://www.youtube-nocookie.com/embed/" + videoid,
                 'thumbnail': thumbnail,
             }
         )

searx/engines/youtube_noapi.py

@@ -32,12 +32,6 @@ time_range_url = '&sp=EgII{time_range}%253D%253D'
 next_page_url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
 time_range_dict = {'day': 'Ag', 'week': 'Aw', 'month': 'BA', 'year': 'BQ'}
 
-embedded_url = (
-    '<iframe width="540" height="304" '
-    + 'data-src="https://www.youtube-nocookie.com/embed/{videoid}" '
-    + 'frameborder="0" allowfullscreen></iframe>'
-)
-
 base_youtube_url = 'https://www.youtube.com/watch?v='
 
 
@@ -91,7 +85,7 @@ def parse_next_page_response(response_text):
                 'author': section['ownerText']['runs'][0]['text'],
                 'length': section['lengthText']['simpleText'],
                 'template': 'videos.html',
-                'embedded': embedded_url.format(videoid=section['videoId']),
+                'iframe_src': 'https://www.youtube-nocookie.com/embed/' + section['videoId'],
                 'thumbnail': section['thumbnail']['thumbnails'][-1]['url'],
             }
         )
@@ -150,7 +144,6 @@ def parse_first_page_response(response_text):
                 thumbnail = 'https://i.ytimg.com/vi/' + videoid + '/hqdefault.jpg'
                 title = get_text_from_json(video.get('title', {}))
                 content = get_text_from_json(video.get('descriptionSnippet', {}))
-                embedded = embedded_url.format(videoid=videoid)
                 author = get_text_from_json(video.get('ownerText', {}))
                 length = get_text_from_json(video.get('lengthText', {}))
 
@@ -163,7 +156,7 @@ def parse_first_page_response(response_text):
                         'author': author,
                         'length': length,
                         'template': 'videos.html',
-                        'embedded': embedded,
+                        'iframe_src': 'https://www.youtube-nocookie.com/embed/' + videoid,
                         'thumbnail': thumbnail,
                     }
                 )

searx/plugins/hostname_replace.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import re
-from urllib.parse import urlunparse
+from urllib.parse import urlunparse, urlparse
 from searx import settings
 from searx.plugins import logger
 from flask_babel import gettext
@@ -17,16 +17,30 @@ replacements = {re.compile(p): r for (p, r) in settings[plugin_id].items()} if p
 
 logger = logger.getChild(plugin_id)
 parsed = 'parsed_url'
+_url_fields = ['iframe_src', 'audio_src']
 
 
 def on_result(request, search, result):
-    if parsed not in result:
-        return True
+
     for (pattern, replacement) in replacements.items():
-        if pattern.search(result[parsed].netloc):
-            if not replacement:
-                return False
-            result[parsed] = result[parsed]._replace(netloc=pattern.sub(replacement, result[parsed].netloc))
-            result['url'] = urlunparse(result[parsed])
+
+        if parsed in result:
+            if pattern.search(result[parsed].netloc):
+                # to keep or remove this result from the result list depends
+                # (only) on the 'parsed_url'
+                if not replacement:
+                    return False
+                result[parsed] = result[parsed]._replace(netloc=pattern.sub(replacement, result[parsed].netloc))
+                result['url'] = urlunparse(result[parsed])
+
+        for url_field in _url_fields:
+            if result.get(url_field):
+                url_src = urlparse(result[url_field])
+                if pattern.search(url_src.netloc):
+                    if not replacement:
+                        del result[url_field]
+                    else:
+                        url_src = url_src._replace(netloc=pattern.sub(replacement, url_src.netloc))
+                        result[url_field] = urlunparse(url_src)
 
     return True

searx/static/themes/simple/src/less/embedded.less

@@ -0,0 +1,19 @@
+iframe[src^="https://w.soundcloud.com"] {
+  height: 120px;
+}
+
+iframe[src^="https://www.deezer.com"] {
+  // The real size is 92px, but 94px are needed to avoid an inner scrollbar of
+  // the embedded HTML.
+  height: 94px;
+}
+
+iframe[src^="https://www.mixcloud.com"] {
+  // the embedded player from mixcloud has some quirks: initial there is an
+  // issue with an image URL that is blocked since it is an a Cross-Origin
+  // request. The alternative text (<img alt='Mixcloud Logo'> then cause an
+  // scrollbar in the inner of the iframe we can't avoid.  Another quirk comes
+  // when pressing the play button, somtimes the shown player has an height of
+  // 200px, somtimes 250px.
+  height: 250px;
+}

searx/static/themes/simple/src/less/style.less

@@ -17,6 +17,7 @@
 @import "autocomplete.less";
 @import "detail.less";
 @import "animations.less";
+@import "embedded.less";
 
 // for index.html template
 @import "index.less";
@@ -296,10 +297,34 @@ article[data-vim-selected].category-social {
   padding: 0 5px 25px 0 !important;
 }
 
+.audio-control audio {
+  width: 100%;
+  padding: 10px 0 0 0;
+}
+
+.embedded-content iframe {
+  width: 100%;
+  padding: 10px 0 0 0;
+}
+
 .result-videos .content {
   overflow: hidden;
 }
 
+.result-videos .embedded-video iframe {
+  width: 100%;
+  aspect-ratio: 16 / 9;
+  padding: 10px 0 0 0;
+}
+
+@supports not (aspect-ratio: 1 / 1) {
+  // support older browsers which do not have aspect-ratio
+  // https://caniuse.com/?search=aspect-ratio
+  .result-videos .embedded-video iframe {
+    height: calc(@results-width * 9 / 16);
+  }
+}
+
 .engines {
   .ltr-float-right();
   color: var(--color-result-engines-font);

searx/templates/oscar/result_templates/default.html

@@ -3,13 +3,13 @@
 {{- result_header(result, favicons, loop.index) -}}
 {{- result_sub_header(result, loop.index) -}}
 
-{%- if result.embedded -%}
+{%- if result.iframe_src -%}
     <small> &bull; <a class="text-info btn-collapse collapsed cursor-pointer media-loader disabled_if_nojs" data-toggle="collapse" data-target="#result-media-{{ index }}" data-btn-text-collapsed="{{ _('show media') }}" data-btn-text-not-collapsed="{{ _('hide media') }}" aria-labelledby="result-{{loop.index}}">{{ icon('music') }} {{ _('show media') }}</a></small>
 {%- endif -%}
 
-{%- if result.embedded -%}
-<div id="result-media-{{ index }}" class="collapse">
-   {{- result.embedded|safe -}}
+{% if result.iframe_src -%}
+<div id="result-media-{{ index }}" class="embedded-content invisible">
+  <iframe data-src="{{result.iframe_src}}" frameborder="0" scrolling="no" allowfullscreen></iframe>
 </div>
 {%- endif -%}
 
@@ -24,6 +24,13 @@
 {%- if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif -%}
 {%- endif -%}
 
+{% if result.audio_src -%}
+<div id="result-media-{{ index }}" class="audio-control">
+  <audio controls><source src="{{result.audio_src}}"></audio>
+</div>
+{%- endif %}
+
+
 {%- if rtl -%}
 {{ result_footer_rtl(result, loop.index) }}
 {%- else -%}

searx/templates/oscar/result_templates/videos.html

@@ -3,15 +3,15 @@
 {{- result_header(result, favicons, loop.index) -}}
 {{- result_sub_header(result, loop.index) -}}
 
-{%- if result.embedded -%}
+{%- if result.iframe_src -%}
     <small> &bull; <a class="text-info btn-collapse collapsed cursor-pointer media-loader disabled_if_nojs" data-toggle="collapse" data-target="#result-video-{{ index }}" data-btn-text-collapsed="{{ _('show video') }}" data-btn-text-not-collapsed="{{ _('hide video') }}" aria-labelledby="result-{{loop.index}}">{{ icon('film') }} {{ _('show video') }}</a></small>
 {%- endif -%}
 
-{%- if result.embedded -%}
-<div id="result-video-{{ index }}" class="collapse">
-   {{- result.embedded|safe -}}
+{% if result.iframe_src -%}
+<div id="result-video-{{ index }}" class="embedded-video collapse">
+  <iframe data-src="{{result.iframe_src}}" width="540" height="304" frameborder="0" scrolling="no" allowfullscreen></iframe>
 </div>
-{%- endif -%}
+{%- endif %}
 
 <div class="container-fluid">{{- "" -}}
     <div class="row">{{- "" -}}

searx/templates/simple/result_templates/default.html

@@ -2,7 +2,7 @@
 
 {{ result_header(result, favicons, image_proxify) -}}
 {{- result_sub_header(result) -}}
-{% if result.embedded -%}
+{% if result.iframe_src -%}
 <p class="altlink"> &bull; <a class="btn-collapse collapsed media-loader disabled_if_nojs" data-target="#result-media-{{ index }}" data-btn-text-collapsed="{{ _('show media') }}" data-btn-text-not-collapsed="{{ _('hide media') }}">{{ icon('music-note') }} {{ _('show media') }}</a></p>
 {%- endif %}
 {%- if result.content %}
@@ -15,9 +15,14 @@
   </p>
 {% endif -%}
 {{- result_sub_footer(result, proxify) -}}
-{% if result.embedded -%}
-<div id="result-media-{{ index }}" class="invisible">
-   {{- result.embedded|safe -}}
+{% if result.iframe_src -%}
+<div id="result-media-{{ index }}" class="embedded-content invisible">
+  <iframe data-src="{{result.iframe_src}}" frameborder="0" allowfullscreen></iframe>
+</div>
+{%- endif %}
+{% if result.audio_src -%}
+<div id="result-media-{{ index }}" class="audio-control">
+  <audio controls><source src="{{result.audio_src}}"></audio>
 </div>
 {%- endif %}
 {{- result_footer(result) }}

searx/templates/simple/result_templates/videos.html

@@ -2,7 +2,7 @@
 
 {{ result_header(result, favicons, image_proxify) }}
 {{ result_sub_header(result) }}
-{% if result.embedded -%}
+{% if result.iframe_src -%}
 <p class="altlink"> &bull; <a class="btn-collapse collapsed media-loader disabled_if_nojs" data-target="#result-video-{{ index }}" data-btn-text-collapsed="{{ _('show video') }}" data-btn-text-not-collapsed="{{ _('hide video') }}">{{ icon('film-outline') }} {{ _('show video') }}</a></p>
 {%- endif %}
 {%- if result.content %}
@@ -16,9 +16,9 @@
 {% endif -%}
 </p>
 {{- result_sub_footer(result, proxify) -}}
-{% if result.embedded -%}
-<div id="result-video-{{ index }}" class="invisible">
-   {{- result.embedded|safe -}}
+{% if result.iframe_src -%}
+<div id="result-video-{{ index }}" class="embedded-video invisible">
+  <iframe data-src="{{result.iframe_src}}" frameborder="0" allowfullscreen></iframe>
 </div>
 {%- endif %}
 {{ result_footer(result) }}