From 889b8cb372f6519a2486ebbfc3b5ff837cb84180 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Fri, 31 Mar 2023 13:18:37 +0200 Subject: [PATCH] Don't copy per version anti features to all versions make_v0() modified the apps data structure to copy an anti feature for a specific version to all versions resulting in index-v1 and -v2 to contain wrong anti feature annotations. This patch fixes this and adds a test that the data structure is no longer modified. The bug shadowed bugs in the AF implementation of -v1 and -v2 resulting in not coping the version specific data. This is corrected as well. This is also tested now. For -v2 the AF dict is now sorted to make the result reproducible. Finally The NoSourceSince AF was added as a per version and overall AF in -v1 and is now only applied as an overall AF and the test is updated accordingly. --- fdroidserver/index.py | 33 ++++++++----- fdroidserver/update.py | 3 -- tests/index.TestCase | 5 +- tests/metadata/com.politedroid.yml | 2 + tests/metadata/dump/com.politedroid.yaml | 2 +- tests/repo/entry.json | 4 +- tests/repo/index-v1.json | 15 ++---- tests/repo/index-v2.json | 62 +++++++++++------------- tests/repo/index.xml | 2 +- tests/run-tests | 2 + 10 files changed, 63 insertions(+), 67 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 1ac306c0..7eacb6ee 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -668,20 +668,24 @@ def convert_version(version, app, repodir): else: manifest[en].append({"name": perm[0]}) + antiFeatures = dict() if "AntiFeatures" in app and app["AntiFeatures"]: - ver["antiFeatures"] = {} for antif in app["AntiFeatures"]: # TODO: get reasons from fdroiddata # ver["antiFeatures"][antif] = {"en-US": "reason"} - ver["antiFeatures"][antif] = {} + antiFeatures[antif] = dict() - if "AntiFeatures" in version and version["AntiFeatures"]: - if "antiFeatures" not in ver: - ver["antiFeatures"] = {} - for antif in version["AntiFeatures"]: + if "antiFeatures" in version and version["antiFeatures"]: + for antif in version["antiFeatures"]: # TODO: get reasons from fdroiddata # ver["antiFeatures"][antif] = {"en-US": "reason"} - ver["antiFeatures"][antif] = {} + antiFeatures[antif] = dict() + + if app.get("NoSourceSince"): + antiFeatures["NoSourceSince"] = dict() + + if antiFeatures: + ver["antiFeatures"] = dict(sorted(antiFeatures.items())) if "versionCode" in version: if version["versionCode"] > app["CurrentVersionCode"]: @@ -920,9 +924,11 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ for ikey, iname in sorted(lvalue.items()): lordered[lkey][ikey] = iname app['localized'] = lordered - antiFeatures = app.get('AntiFeatures') + antiFeatures = app.get('antiFeatures', []) + if apps[app["packageName"]].get("NoSourceSince"): + antiFeatures.append("NoSourceSince") if antiFeatures: - app['AntiFeatures'] = sorted(set(antiFeatures)) + app['antiFeatures'] = sorted(set(antiFeatures)) output_packages = collections.OrderedDict() output['packages'] = output_packages @@ -1191,10 +1197,13 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing # doesn't have to do any work by default... apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True) + antiFeatures = list(app.AntiFeatures) if 'antiFeatures' in apklist[0]: - app.AntiFeatures.extend(apklist[0]['antiFeatures']) - if app.AntiFeatures: - afout = sorted(set(app.AntiFeatures)) + antiFeatures.extend(apklist[0]['antiFeatures']) + if app.get("NoSourceSince"): + antiFeatures.append("NoSourceSince") + if antiFeatures: + afout = sorted(set(antiFeatures)) addElementNonEmpty('antifeatures', ','.join(afout), doc, apel) # Check for duplicates - they will make the client unhappy... diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 6168ecb8..f7c055b9 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1821,9 +1821,6 @@ def apply_info_from_latest_apk(apps, apks): bestver = apk['versionCode'] bestapk = apk - if app.get('NoSourceSince'): - apk['antiFeatures'].add('NoSourceSince') - if not app['added']: logging.debug("Don't know when " + appid + " was added") if not app['lastUpdated']: diff --git a/tests/index.TestCase b/tests/index.TestCase index 50c678af..30ec0e12 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import copy import datetime import inspect import logging @@ -425,6 +426,7 @@ class IndexTest(unittest.TestCase): app['icon'] = 'info.zwanenburg.caffeinetile.4.xml' app['CurrentVersionCode'] = 4 apps = {app.id: app} + orig_apps = copy.deepcopy(apps) apk = { 'hash': 'dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db', 'hashType': 'sha256', @@ -436,7 +438,7 @@ class IndexTest(unittest.TestCase): '-1': 'res/drawable/ic_coffee_on.xml', }, 'icons': {'160': 'info.zwanenburg.caffeinetile.4.xml'}, - 'antiFeatures': [], + 'antiFeatures': ['KnownVuln'], 'packageName': 'info.zwanenburg.caffeinetile', 'versionCode': 4, 'name': 'Caffeine Tile', @@ -463,6 +465,7 @@ class IndexTest(unittest.TestCase): ) ) self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml'))) + self.assertEqual(orig_apps, apps, "apps was modified when building the index") def test_v0_invalid_config_exception(self): """Index v0 needs additional config values when using --nosign diff --git a/tests/metadata/com.politedroid.yml b/tests/metadata/com.politedroid.yml index c138b6c8..a1ddc8bf 100644 --- a/tests/metadata/com.politedroid.yml +++ b/tests/metadata/com.politedroid.yml @@ -1,3 +1,5 @@ +AntiFeatures: + - NonFreeNet Categories: - Time License: GPL-3.0-only diff --git a/tests/metadata/dump/com.politedroid.yaml b/tests/metadata/dump/com.politedroid.yaml index 4ecb3346..ec9903bd 100644 --- a/tests/metadata/dump/com.politedroid.yaml +++ b/tests/metadata/dump/com.politedroid.yaml @@ -1,5 +1,5 @@ AllowedAPKSigningKeys: [] -AntiFeatures: [] +AntiFeatures: ['NonFreeNet'] ArchivePolicy: 4 versions AuthorEmail: null AuthorName: null diff --git a/tests/repo/entry.json b/tests/repo/entry.json index f7c1420b..245fb94e 100644 --- a/tests/repo/entry.json +++ b/tests/repo/entry.json @@ -3,9 +3,9 @@ "version": 20002, "index": { "name": "/index-v2.json", - "sha256": "da1d651eb7bbc27d2334819c591baa5afb70b01397849de515dade624a96de6d", + "sha256": "2f92210a7c7f2f3d855006979ebce4dda203de5ae6596a783aa531f8086e2694", "size": 32946, "numPackages": 10 }, "diffs": {} -} \ No newline at end of file +} diff --git a/tests/repo/index-v1.json b/tests/repo/index-v1.json index 74d9cae6..bf2f53ae 100644 --- a/tests/repo/index-v1.json +++ b/tests/repo/index-v1.json @@ -170,7 +170,8 @@ }, { "antiFeatures": [ - "NoSourceSince" + "NoSourceSince", + "NonFreeNet" ], "categories": [ "Time" @@ -230,9 +231,6 @@ "com.politedroid": [ { "added": 1498176000000, - "antiFeatures": [ - "NoSourceSince" - ], "apkName": "com.politedroid_6.apk", "hash": "70c2f776a2bac38a58a7d521f96ee0414c6f0fb1de973c3ca8b10862a009247d", "hashType": "sha256", @@ -257,9 +255,6 @@ }, { "added": 1498176000000, - "antiFeatures": [ - "NoSourceSince" - ], "apkName": "com.politedroid_5.apk", "hash": "5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d", "hashType": "sha256", @@ -284,9 +279,6 @@ }, { "added": 1498176000000, - "antiFeatures": [ - "NoSourceSince" - ], "apkName": "com.politedroid_4.apk", "hash": "c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075", "hashType": "sha256", @@ -324,7 +316,6 @@ "added": 1498176000000, "antiFeatures": [ "KnownVuln", - "NoSourceSince", "NonFreeAssets", "UpstreamNonFree" ], @@ -698,4 +689,4 @@ } ] } -} \ No newline at end of file +} diff --git a/tests/repo/index-v2.json b/tests/repo/index-v2.json index 536abd6f..a84be25d 100644 --- a/tests/repo/index-v2.json +++ b/tests/repo/index-v2.json @@ -72,8 +72,7 @@ "file": { "name": "/com.politedroid_6.apk", "sha256": "70c2f776a2bac38a58a7d521f96ee0414c6f0fb1de973c3ca8b10862a009247d", - "size": 16578, - "ipfsCIDv1": "bafybeidvgxrq77qr7yqkcnykdfvszsxjqc5kzt6ya5k7r666wriadrylt4" + "size": 16578 }, "manifest": { "versionName": "1.5", @@ -97,7 +96,8 @@ ] }, "antiFeatures": { - "NoSourceSince": {} + "NoSourceSince": {}, + "NonFreeNet": {} } }, "5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d": { @@ -105,8 +105,7 @@ "file": { "name": "/com.politedroid_5.apk", "sha256": "5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d", - "size": 18817, - "ipfsCIDv1": "bafybeifbrio5rumqvgfd5sihs7yihux2yktfvd5i7jimlgrwchzcvi6ldu" + "size": 18817 }, "manifest": { "versionName": "1.4", @@ -130,7 +129,8 @@ ] }, "antiFeatures": { - "NoSourceSince": {} + "NoSourceSince": {}, + "NonFreeNet": {} } }, "c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075": { @@ -138,8 +138,7 @@ "file": { "name": "/com.politedroid_4.apk", "sha256": "c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075", - "size": 18489, - "ipfsCIDv1": "bafybeicridbev22c2rt3lwbfsrkafcf3yepak7kpvk6zgbayrxls2mmwim" + "size": 18489 }, "manifest": { "versionName": "1.3", @@ -172,7 +171,8 @@ ] }, "antiFeatures": { - "NoSourceSince": {} + "NoSourceSince": {}, + "NonFreeNet": {} } }, "665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d": { @@ -180,8 +180,7 @@ "file": { "name": "/com.politedroid_3.apk", "sha256": "665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d", - "size": 17552, - "ipfsCIDv1": "bafybeib7arokhivttalcnq5ieu5fx5pzn7vo5qpmdiozqodzhb4ba53nd4" + "size": 17552 }, "manifest": { "versionName": "1.2", @@ -214,7 +213,11 @@ ] }, "antiFeatures": { - "NoSourceSince": {} + "KnownVuln": {}, + "NoSourceSince": {}, + "NonFreeAssets": {}, + "NonFreeNet": {}, + "UpstreamNonFree": {} } } } @@ -247,8 +250,7 @@ "file": { "name": "/duplicate.permisssions_9999999.apk", "sha256": "8367857fe75f85321ce2c344b34804d0bc193707f6ba03710d025d9030803434", - "size": 27446, - "ipfsCIDv1": "bafybeicucr4lk7fynyde4fpxubudpl6m6wqnuq2j6vjroutjyryw24en3u" + "size": 27446 }, "manifest": { "versionName": "", @@ -390,8 +392,7 @@ "file": { "name": "/urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk", "sha256": "15c0ec72c74a3791f42cdb43c57df0fb11a4dbb656851bbb8cf05b26a8372789", - "size": 11471, - "ipfsCIDv1": "bafybeig77jwqx243si3gh55iqx4gkcxhltkt6pjimzgigfsk3kshsi6qem" + "size": 11471 }, "manifest": { "versionName": "0.1", @@ -430,8 +431,7 @@ "file": { "name": "/info.zwanenburg.caffeinetile_4.apk", "sha256": "dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db", - "size": 11740, - "ipfsCIDv1": "bafybeigormhkorw3mk6pkkfk63kkmxpvwylthgj67geulvskc5acr65sym" + "size": 11740 }, "manifest": { "versionName": "1.3", @@ -482,8 +482,7 @@ "file": { "name": "/no.min.target.sdk_987.apk", "sha256": "e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c", - "size": 14102, - "ipfsCIDv1": "bafybeidwxseoagnew3gtlasttqovl7ciuwxaud5a5p4a5pzpbrfcfj2gaa" + "size": 14102 }, "manifest": { "versionName": "1.2-fake", @@ -541,8 +540,7 @@ "file": { "name": "/obb.main.oldversion_1444412523.apk", "sha256": "c5f149e526f89c05c62923bdb7bb1e2be5673c46ec85143f41e514340631449c", - "size": 14323, - "ipfsCIDv1": "bafybeicnwnpiyfke3tbk3nve62meig65vved34i6kesjkksdciff6242ui" + "size": 14323 }, "obbMainFile": { "name": "/main.1434483388.obb.main.oldversion.obb", @@ -639,8 +637,7 @@ "file": { "name": "/obb.main.twoversions_1101617.apk", "sha256": "9bc74566f089ef030ac33e7fbd99d92f1a38f363fb499fed138d9e7b774e821c", - "size": 11481, - "ipfsCIDv1": "bafybeiblpfmwololxgsrum337rbbbsqg2gk6hytvt6szf4njubosju3bme" + "size": 11481 }, "src": { "name": "/obb.main.twoversions_1101617_src.tar.gz", @@ -671,8 +668,7 @@ "file": { "name": "/obb.main.twoversions_1101615.apk", "sha256": "7b0b7b9ba248e15751a16e3a0e01e1e24cbb673686c38422030cb75d5c33f0bb", - "size": 11480, - "ipfsCIDv1": "bafybeigglr3iefb3es4lp2sgfacppk3w2qqtuykjgf4actebpalyizef3q" + "size": 11480 }, "obbMainFile": { "name": "/main.1101615.obb.main.twoversions.obb", @@ -698,8 +694,7 @@ "file": { "name": "/obb.main.twoversions_1101613.apk", "sha256": "cce97a52ff18d843185be7f22ecb1a557c36b7a9f8ba07a8be94e328e00b35dc", - "size": 11477, - "ipfsCIDv1": "bafybeicocjo4khzp2rkui2ltvrhbksrm373lr3pb43ut7hqgbllfjpv6ti" + "size": 11477 }, "obbMainFile": { "name": "/main.1101613.obb.main.twoversions.obb", @@ -777,8 +772,7 @@ "file": { "name": "/obb.mainpatch.current_1619.apk", "sha256": "eda5fc3ecfdac3252717e36bdbc9820865baeef162264af9ba5db7364f0e7a0c", - "size": 11479, - "ipfsCIDv1": "bafybeievo4e234mllujityvtjgeltauyfbriszoqddzygmimcm4mo3zyqu" + "size": 11479 }, "obbMainFile": { "name": "/main.1619.obb.mainpatch.current.obb", @@ -809,8 +803,7 @@ "file": { "name": "/obb.mainpatch.current_1619_another-release-key.apk", "sha256": "42e7d6d2f8254aaf9fe95ba6ecc233ee8c3cd543a3e4f3f9ebe1b638221122fa", - "size": 10541, - "ipfsCIDv1": "bafybeiatdbzlxairqzvdowevwuy7nk24rknc55jpip2wb2sq4c3f7mtngm" + "size": 10541 }, "obbMainFile": { "name": "/main.1619.obb.mainpatch.current.obb", @@ -877,8 +870,7 @@ "file": { "name": "/souch.smsbypass_9.apk", "sha256": "80b0ae68a1189baa3ee6717092e3dbf1a4210165f7f7e5f2f9616bd63a2ec01d", - "size": 81295, - "ipfsCIDv1": "bafybeihaccfnt32q2iwfulh2m7jvdivuunlw6t72wa7jfi7igxvqxjqszy" + "size": 81295 }, "manifest": { "versionName": "0.9", @@ -917,4 +909,4 @@ } } } -} \ No newline at end of file +} diff --git a/tests/repo/index.xml b/tests/repo/index.xml index 85dffe5d..dfdb76b6 100644 --- a/tests/repo/index.xml +++ b/tests/repo/index.xml @@ -318,7 +318,7 @@ APK is called F-Droid Privileged Extension. https://github.com/miguelvps/PoliteDroid/issues 1.5 6 - NoSourceSince + NoSourceSince,NonFreeNet 1.5 6 diff --git a/tests/run-tests b/tests/run-tests index 926e53ed..ac00e193 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -306,6 +306,8 @@ $sed -i.tmp -e 's,timestamp="[0-9]*",timestamp="1676634233",' repo/index.xml diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json diff -uw $WORKSPACE/tests/repo/index-v1.json repo/index-v1.json +sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.json +diff -uw $WORKSPACE/tests/repo/index-v2.json repo/index-v2.json #------------------------------------------------------------------------------#