From 0ad45a94a81a4d48207f14222261a4b9301dca71 Mon Sep 17 00:00:00 2001 From: pmmayero Date: Wed, 14 Sep 2022 06:42:23 +0000 Subject: [PATCH] Addition of IPFS CIDv1 to Index IPFS CIDv1 is only generated for APKs and "repo files" --- .gitlab-ci.yml | 1 + fdroidserver/common.py | 19 +++++++++++++++++++ fdroidserver/index.py | 6 +++++- fdroidserver/update.py | 6 +++++- tests/common.TestCase | 14 ++++++++++++++ .../apk/info.guardianproject.urzip.yaml | 1 + tests/metadata/apk/org.dyndns.fules.ck.yaml | 1 + tests/repo/index-v1.json | 2 +- tests/repo/index.xml | 2 +- tests/update.TestCase | 14 ++++++++++++-- 10 files changed, 60 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a04d6dd..e9f70075 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,6 +92,7 @@ debian_testing: fdroidserver git gnupg + ipfs-cid python3-defusedxml python3-setuptools zipalign diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 6e2ae762..16315376 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4118,6 +4118,25 @@ def run_yamllint(path, indent=0): return '\n'.join(result) +def calculate_IPFS_cid(filename): + """ + Calculate the IPFS CID of a file and add it to the index. + + uses ipfs_cid package at https://packages.debian.org/sid/ipfs-cid + Returns CIDv1 of a file as per IPFS recommendation + """ + exe_name = 'ipfs_cid' + if not set_command_in_config(exe_name) or not config.get(exe_name): + logging.info(_("%s not found, skipping CIDv1 generation") % exe_name) + return + file_cid = subprocess.run([config[exe_name], filename], capture_output=True) + + if file_cid.returncode == 0: + cid_output = file_cid.stdout.decode() + cid_output_dict = json.loads(cid_output) + return cid_output_dict['CIDv1'] + + def sha256sum(filename): """Calculate the sha256 of the given file.""" sha = hashlib.sha256() diff --git a/fdroidserver/index.py b/fdroidserver/index.py index bdb1dd86..24f8a56e 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -607,6 +607,10 @@ def convert_version(version, app, repodir): "size": version["size"] } + ipfsCIDv1 = version.get("ipfsCIDv1") + if ipfsCIDv1: + ver["file"]["ipfsCIDv1"] = ipfsCIDv1 + if "srcname" in version: ver["src"] = file_entry(os.path.join(repodir, version["srcname"])) @@ -945,7 +949,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ for k, v in sorted(package.items()): if not v: continue - if k in ('icon', 'icons', 'icons_src', 'name', ): + if k in ('icon', 'icons', 'icons_src', 'ipfsCIDv1', 'name'): continue d[k] = v diff --git a/fdroidserver/update.py b/fdroidserver/update.py index c957dab4..e6bea4bf 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -58,7 +58,7 @@ if hasattr(Image, 'DecompressionBombWarning'): warnings.simplefilter('error', Image.DecompressionBombWarning) Image.MAX_IMAGE_PIXELS = 0xffffff # 4096x4096 -METADATA_VERSION = 20001 +METADATA_VERSION = 20002 # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE UNSET_VERSION_CODE = -0x100000000 @@ -1148,6 +1148,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_file['apkName'] = name_utf8 repo_file['hash'] = shasum repo_file['hashType'] = 'sha256' + repo_file['ipfsCIDv1'] = common.calculate_IPFS_cid(name_utf8) repo_file['versionCode'] = 0 repo_file['versionName'] = shasum[0:7] # the static ID is the SHA256 unless it is set in the metadata @@ -1212,6 +1213,9 @@ def scan_apk(apk_file, require_signature=True): 'icons': {}, 'antiFeatures': set(), } + ipfsCIDv1 = common.calculate_IPFS_cid(apk_file) + if ipfsCIDv1: + apk['ipfsCIDv1'] = ipfsCIDv1 scan_apk_androguard(apk, apk_file) diff --git a/tests/common.TestCase b/tests/common.TestCase index 549167fc..470e3443 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1508,6 +1508,20 @@ class CommonTest(unittest.TestCase): with self.assertRaises(SyntaxError): fdroidserver.common.calculate_math_string('1-1 # no comment') + def test_calculate_IPFS_cid_with_no_tool(self): + fdroidserver.common.config = {'ipfs_cid': None} + self.assertIsNone(fdroidserver.common.calculate_IPFS_cid('urzip.apk')) + self.assertIsNone(fdroidserver.common.calculate_IPFS_cid('FileDoesNotExist')) + + @unittest.skipUnless(shutil.which('ipfs_cid'), 'calculate_IPFS_cid needs ipfs_cid') + def test_calculate_IPFS_cid(self): + fdroidserver.common.config = dict() + self.assertIsNone(fdroidserver.common.calculate_IPFS_cid('FileDoesNotExist')) + self.assertEqual( + fdroidserver.common.calculate_IPFS_cid('urzip.apk'), + "bafybeigmtgrwyvj77jaflje2rf533haeqtpu2wtwsctryjusjnsawacsam", + ) + def test_deploy_build_log_with_rsync_with_id_file(self): mocklogcontent = bytes( diff --git a/tests/metadata/apk/info.guardianproject.urzip.yaml b/tests/metadata/apk/info.guardianproject.urzip.yaml index 4632c314..a3e75d87 100644 --- a/tests/metadata/apk/info.guardianproject.urzip.yaml +++ b/tests/metadata/apk/info.guardianproject.urzip.yaml @@ -9,6 +9,7 @@ icons: icons_src: '-1': res/drawable/ic_launcher.png '160': res/drawable/ic_launcher.png +ipfsCIDv1: bafybeigmtgrwyvj77jaflje2rf533haeqtpu2wtwsctryjusjnsawacsam minSdkVersion: 4 name: urzip packageName: info.guardianproject.urzip diff --git a/tests/metadata/apk/org.dyndns.fules.ck.yaml b/tests/metadata/apk/org.dyndns.fules.ck.yaml index 23a0325f..b4e16511 100644 --- a/tests/metadata/apk/org.dyndns.fules.ck.yaml +++ b/tests/metadata/apk/org.dyndns.fules.ck.yaml @@ -13,6 +13,7 @@ icons_src: '120': res/drawable-ldpi-v4/icon_launcher.png '160': res/drawable-mdpi-v4/icon_launcher.png '240': res/drawable-hdpi-v4/icon_launcher.png +ipfsCIDv1: bafybeifijmr5ygvfvig4vzbmdc3ysj6m46ddohaol4vgp4qoyooqpc27zu minSdkVersion: 7 name: Compass Keyboard nativecode: diff --git a/tests/repo/index-v1.json b/tests/repo/index-v1.json index b6b870bc..409f333a 100644 --- a/tests/repo/index-v1.json +++ b/tests/repo/index-v1.json @@ -1,7 +1,7 @@ { "repo": { "timestamp": 1502845383782, - "version": 20001, + "version": 20002, "name": "My First F-Droid Repo Demo", "icon": "icon.png", "address": "https://MyFirstFDroidRepo.org/fdroid/repo", diff --git a/tests/repo/index.xml b/tests/repo/index.xml index 78fd1c99..7c9a1cd6 100644 --- a/tests/repo/index.xml +++ b/tests/repo/index.xml @@ -1,6 +1,6 @@ - + This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid. http://foobarfoobarfoobar.onion/fdroid/repo https://foo.bar/fdroid/repo diff --git a/tests/update.TestCase b/tests/update.TestCase index 736a50b7..7e7f8583 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -836,7 +836,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.config = config apk_info = fdroidserver.update.scan_apk('repo/no.min.target.sdk_987.apk') self.maxDiff = None - self.assertDictEqual(apk_info, { + expected = { 'icons': {}, 'icons_src': {'-1': 'res/drawable/ic_launcher.png', '160': 'res/drawable/ic_launcher.png'}, @@ -859,11 +859,18 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.UsesPermission(name='android.permission.READ_PHONE_STATE', maxSdkVersion=None), fdroidserver.update.UsesPermission(name='android.permission.READ_EXTERNAL_STORAGE', - maxSdkVersion=None)]}) + maxSdkVersion=None), + ], + } + if config.get('ipfs_cid'): + expected['ipfsCIDv1'] = 'bafybeidwxseoagnew3gtlasttqovl7ciuwxaud5a5p4a5pzpbrfcfj2gaa' + + self.assertDictEqual(apk_info, expected) def test_scan_apk_no_sig(self): config = dict() fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) if os.path.basename(os.getcwd()) != 'tests': @@ -929,6 +936,7 @@ class UpdateTest(unittest.TestCase): config = dict() fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config fdroidserver.update.config = config os.chdir(os.path.join(localmodule, 'tests')) @@ -986,6 +994,8 @@ class UpdateTest(unittest.TestCase): with open(savepath, 'r') as f: from_yaml = yaml.load(f, Loader=TestLoader) self.maxDiff = None + if not config.get('ipfs_cid'): + del from_yaml['ipfsCIDv1'] # handle when ipfs_cid is not installed self.assertEqual(apk, from_yaml) def test_process_apk_signed_by_disabled_algorithms(self):