From 56bed02a29b51152556c80bc8fd31f4557f6a47f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 21 Feb 2024 18:01:56 +0100 Subject: [PATCH] install: download_apk() fetchs APKs by appid based on the index --- fdroidserver/common.py | 93 +++++++++++++++++++++++++++++++++++++++++ fdroidserver/install.py | 45 +++++++++++++++++++- tests/common.TestCase | 16 +++++++ 3 files changed, 152 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7553f74b..5d54ac09 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -28,6 +28,7 @@ # common.py is imported by all modules, so do not import third-party # libraries here as they will become a requirement for all commands. +import copy import difflib from typing import List import git @@ -60,6 +61,7 @@ from base64 import urlsafe_b64encode from binascii import hexlify from datetime import datetime, timedelta, timezone from queue import Queue +from urllib.parse import urlparse, urlunparse from zipfile import ZipFile import fdroidserver.metadata @@ -435,6 +437,14 @@ def get_config(): return config +def get_cachedir(): + cachedir = config and config.get('cachedir') + if cachedir and os.path.exists(cachedir): + return Path(cachedir) + else: + return Path(tempfile.mkdtemp()) + + def regsub_file(pattern, repl, path): with open(path, 'rb') as f: text = f.read() @@ -609,6 +619,17 @@ def parse_mirrors_config(mirrors): raise TypeError(_('only accepts strings, lists, and tuples')) +def append_filename_to_mirrors(filename, mirrors): + """Append the filename to all "url" entries in the mirrors dict.""" + appended = copy.deepcopy(mirrors) + for mirror in appended: + parsed = urlparse(mirror['url']) + mirror['url'] = urlunparse( + parsed._replace(path=os.path.join(parsed.path, filename)) + ) + return appended + + def file_entry(filename, hash_value=None): meta = {} meta["name"] = "/" + Path(filename).as_posix().split("/", 1)[1] @@ -4620,3 +4641,75 @@ def _install_ndk(ndk): logging.info( _('Set NDK {release} ({version}) up').format(release=ndk, version=version) ) + + +FDROIDORG_MIRRORS = [ + { + 'isPrimary': True, + 'url': 'https://f-droid.org/repo', + 'dnsA': ['65.21.79.229', '136.243.44.143'], + 'dnsAAAA': ['2a01:4f8:212:c98::2', '2a01:4f9:3b:546d::2'], + 'worksWithoutSNI': True, + }, + { + 'url': 'http://fdroidorg6cooksyluodepej4erfctzk7rrjpjbbr6wx24jh3lqyfwyd.onion/fdroid/repo' + }, + { + 'url': 'http://dotsrccccbidkzg7oc7oj4ugxrlfbt64qebyunxbrgqhxiwj3nl6vcad.onion/fdroid/repo' + }, + { + 'url': 'http://ftpfaudev4triw2vxiwzf4334e3mynz7osqgtozhbc77fixncqzbyoyd.onion/fdroid/repo' + }, + { + 'url': 'http://lysator7eknrfl47rlyxvgeamrv7ucefgrrlhk7rouv3sna25asetwid.onion/pub/fdroid/repo' + }, + { + 'url': 'http://mirror.ossplanetnyou5xifr6liw5vhzwc2g2fmmlohza25wwgnnaw65ytfsad.onion/fdroid/repo' + }, + {'url': 'https://fdroid.tetaneutral.net/fdroid/repo', 'countryCode': 'FR'}, + { + 'url': 'https://ftp.agdsn.de/fdroid/repo', + 'countryCode': 'DE', + "dnsA": ["141.30.235.39"], + "dnsAAAA": ["2a13:dd85:b00:12::1"], + "worksWithoutSNI": True, + }, + { + 'url': 'https://ftp.fau.de/fdroid/repo', + 'countryCode': 'DE', + "dnsA": ["131.188.12.211"], + "dnsAAAA": ["2001:638:a000:1021:21::1"], + "worksWithoutSNI": True, + }, + {'url': 'https://ftp.gwdg.de/pub/android/fdroid/repo', 'countryCode': 'DE'}, + { + 'url': 'https://ftp.lysator.liu.se/pub/fdroid/repo', + 'countryCode': 'SE', + "dnsA": ["130.236.254.251", "130.236.254.253"], + "dnsAAAA": ["2001:6b0:17:f0a0::fb", "2001:6b0:17:f0a0::fd"], + "worksWithoutSNI": True, + }, + {'url': 'https://mirror.cyberbits.eu/fdroid/repo', 'countryCode': 'FR'}, + { + 'url': 'https://mirror.fcix.net/fdroid/repo', + 'countryCode': 'US', + "dnsA": ["23.152.160.16"], + "dnsAAAA": ["2620:13b:0:1000::16"], + "worksWithoutSNI": True, + }, + {'url': 'https://mirror.kumi.systems/fdroid/repo', 'countryCode': 'AT'}, + {'url': 'https://mirror.level66.network/fdroid/repo', 'countryCode': 'DE'}, + {'url': 'https://mirror.ossplanet.net/fdroid/repo', 'countryCode': 'TW'}, + {'url': 'https://mirrors.dotsrc.org/fdroid/repo', 'countryCode': 'DK'}, + {'url': 'https://opencolo.mm.fcix.net/fdroid/repo', 'countryCode': 'US'}, + { + 'url': 'https://plug-mirror.rcac.purdue.edu/fdroid/repo', + 'countryCode': 'US', + "dnsA": ["128.211.151.252"], + "dnsAAAA": ["2001:18e8:804:35::1337"], + "worksWithoutSNI": True, + }, +] +FDROIDORG_FINGERPRINT = ( + '43238D512C1E5EB2D6569F4A3AFBF5523418B82E0A3ED1552770ABB9A9C9CCAB' +) diff --git a/fdroidserver/install.py b/fdroidserver/install.py index b9370ee5..362d5aed 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -20,17 +20,58 @@ import sys import os import glob -from argparse import ArgumentParser import logging +from argparse import ArgumentParser +from pathlib import Path +from urllib.parse import urlencode, urlparse, urlunparse + from . import _ -from . import common +from . import common, index, net from .common import SdkToolsPopen from .exception import FDroidException config = None +DEFAULT_IPFS_GATEWAYS = ("https://gateway.ipfs.io/ipfs/",) + + +def download_apk(appid='org.fdroid.fdroid'): + """Download an APK from F-Droid via the first mirror that works.""" + url = urlunparse( + urlparse(common.FDROIDORG_MIRRORS[0]['url'])._replace( + query=urlencode({'fingerprint': common.FDROIDORG_FINGERPRINT}) + ) + ) + + data, _ignored = index.download_repo_index_v2(url) + app = data.get('packages', dict()).get(appid) + preferred_version = None + for version in app['versions'].values(): + if not preferred_version: + # if all else fails, use the first one + preferred_version = version + if not version.get('releaseChannels'): + # prefer APK in default release channel + preferred_version = version + break + print('skipping', version) + + mirrors = common.append_filename_to_mirrors( + preferred_version['file']['name'][1:], common.FDROIDORG_MIRRORS + ) + ipfsCIDv1 = preferred_version['file'].get('ipfsCIDv1') + if ipfsCIDv1: + for gateway in DEFAULT_IPFS_GATEWAYS: + mirrors.append({'url': os.path.join(gateway, ipfsCIDv1)}) + f = net.download_using_mirrors(mirrors) + if f and os.path.exists(f): + versionCode = preferred_version['manifest']['versionCode'] + f = Path(f) + return str(f.rename(f.with_stem(f'{appid}_{versionCode}')).resolve()) + + def devices(): p = SdkToolsPopen(['adb', "devices"]) if p.returncode != 0: diff --git a/tests/common.TestCase b/tests/common.TestCase index f04df702..c4959cb5 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -36,6 +36,7 @@ if localmodule not in sys.path: sys.path.insert(0, localmodule) import fdroidserver.index +import fdroidserver.install import fdroidserver.signindex import fdroidserver.common import fdroidserver.metadata @@ -2967,6 +2968,21 @@ class CommonTest(unittest.TestCase): knownapks.recordapk(fake_apk, default_date=datetime.now(timezone.utc)) self.assertEqual(knownapks.apks[fake_apk], now) + def test_append_filename_to_mirrors(self): + filename = 'test.apk' + url = 'https://example.com/fdroid/repo' + mirrors = [{'url': url}] + self.assertEqual( + [{'url': url + '/' + filename}], + fdroidserver.common.append_filename_to_mirrors(filename, mirrors), + ) + + def test_append_filename_to_mirrors_full(self): + filename = 'test.apk' + mirrors = fdroidserver.common.FDROIDORG_MIRRORS + for mirror in fdroidserver.common.append_filename_to_mirrors(filename, mirrors): + self.assertTrue(mirror['url'].endswith('/' + filename)) + APKS_WITH_JAR_SIGNATURES = ( (