From ce270961e82585971579844c64d7cde5f5d855ec Mon Sep 17 00:00:00 2001 From: Bnyro Date: Thu, 28 Sep 2023 19:14:30 +0200 Subject: [PATCH] [feat] engine: implementation of mastodon --- docs/dev/engines/online/mastodon.rst | 13 +++++ searx/engines/mastodon.py | 80 ++++++++++++++++++++++++++++ searx/settings.yml | 12 +++++ 3 files changed, 105 insertions(+) create mode 100644 docs/dev/engines/online/mastodon.rst create mode 100644 searx/engines/mastodon.py diff --git a/docs/dev/engines/online/mastodon.rst b/docs/dev/engines/online/mastodon.rst new file mode 100644 index 000000000..dc372f121 --- /dev/null +++ b/docs/dev/engines/online/mastodon.rst @@ -0,0 +1,13 @@ +.. _mastodon engine: + +======== +Mastodon +======== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.engines.mastodon + :members: diff --git a/searx/engines/mastodon.py b/searx/engines/mastodon.py new file mode 100644 index 000000000..1f9d4014f --- /dev/null +++ b/searx/engines/mastodon.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Mastodon_ is an open source alternative to large social media platforms like +Twitter/X, Facebook, ... + +Since it's federated and self-hostable, there's a large amount of available +instances, which can be chosen instead by modifying ``base_url``. + +We use their official API_ for searching, but unfortunately, their Search API_ +forbids pagination without OAuth. + +That's why we use tootfinder.ch for finding posts, which doesn't support searching +for users, accounts or other types of content on Mastodon however. + +.. _Mastodon: https://mastodon.social +.. _API: https://docs.joinmastodon.org/api/ + +""" + +from urllib.parse import urlencode +from datetime import datetime + +about = { + "website": 'https://joinmastodon.org/', + "wikidata_id": 'Q27986619', + "official_api_documentation": 'https://docs.joinmastodon.org/api/', + "use_official_api": True, + "require_api_key": False, + "results": 'JSON', +} +categories = ['social media'] + +base_url = "https://mastodon.social" +mastodon_type = "accounts" + +# https://github.com/searxng/searxng/pull/2857#issuecomment-1741713999 +page_size = 40 + + +def request(query, params): + args = { + 'q': query, + 'resolve': 'false', + 'type': mastodon_type, + 'limit': page_size, + } + params['url'] = f"{base_url}/api/v2/search?{urlencode(args)}" + return params + + +def response(resp): + results = [] + + json = resp.json() + + for result in json[mastodon_type]: + if mastodon_type == "accounts": + results.append( + { + 'url': result['uri'], + 'title': result['username'] + f" ({result['followers_count']} followers)", + 'content': result['note'], + 'thumbnail': result.get('avatar'), + 'publishedDate': datetime.strptime(result['created_at'][:10], "%Y-%m-%d"), + } + ) + elif mastodon_type == "hashtags": + uses_count = sum(int(entry['uses']) for entry in result['history']) + user_count = sum(int(entry['accounts']) for entry in result['history']) + results.append( + { + 'url': result['url'], + 'title': result['name'], + 'content': f"Hashtag has been used {uses_count} times by {user_count} different users", + } + ) + else: + raise ValueError(f"Unsupported mastodon type: {mastodon_type}") + + return results diff --git a/searx/settings.yml b/searx/settings.yml index 0d3c73341..5012f07ad 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1016,6 +1016,18 @@ engines: require_api_key: false results: HTML + - name: mastodon users + engine: mastodon + mastodon_type: accounts + base_url: https://mastodon.social + shortcut: mau + + - name: mastodon hashtags + engine: mastodon + mastodon_type: hashtags + base_url: https://mastodon.social + shortcut: mah + - name: matrixrooms engine: matrixrooms shortcut: mtrx