Merge branch 'fix_antifeatures' into 'master'

Fix antifeatures

See merge request fdroid/fdroidserver!1331

fdroidserver/index.py

@@ -668,20 +668,24 @@ def convert_version(version, app, repodir):
                 else:
                     manifest[en].append({"name": perm[0]})
 
+    antiFeatures = dict()
     if "AntiFeatures" in app and app["AntiFeatures"]:
-        ver["antiFeatures"] = {}
         for antif in app["AntiFeatures"]:
             # TODO: get reasons from fdroiddata
             # ver["antiFeatures"][antif] = {"en-US": "reason"}
-            ver["antiFeatures"][antif] = {}
+            antiFeatures[antif] = dict()
 
-    if "AntiFeatures" in version and version["AntiFeatures"]:
+    if "antiFeatures" in version and version["antiFeatures"]:
-        if "antiFeatures" not in ver:
+        for antif in version["antiFeatures"]:
-            ver["antiFeatures"] = {}
-        for antif in version["AntiFeatures"]:
             # TODO: get reasons from fdroiddata
             # ver["antiFeatures"][antif] = {"en-US": "reason"}
-            ver["antiFeatures"][antif] = {}
+            antiFeatures[antif] = dict()
+
+    if app.get("NoSourceSince"):
+        antiFeatures["NoSourceSince"] = dict()
+
+    if antiFeatures:
+        ver["antiFeatures"] = dict(sorted(antiFeatures.items()))
 
     if "versionCode" in version:
         if version["versionCode"] > app["CurrentVersionCode"]:
@@ -920,9 +924,11 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
                 for ikey, iname in sorted(lvalue.items()):
                     lordered[lkey][ikey] = iname
             app['localized'] = lordered
-        antiFeatures = app.get('AntiFeatures')
+        antiFeatures = app.get('antiFeatures', [])
+        if apps[app["packageName"]].get("NoSourceSince"):
+            antiFeatures.append("NoSourceSince")
         if antiFeatures:
-            app['AntiFeatures'] = sorted(set(antiFeatures))
+            app['antiFeatures'] = sorted(set(antiFeatures))
 
     output_packages = collections.OrderedDict()
     output['packages'] = output_packages
@@ -1191,10 +1197,13 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
         # doesn't have to do any work by default...
         apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 
+        antiFeatures = list(app.AntiFeatures)
         if 'antiFeatures' in apklist[0]:
-            app.AntiFeatures.extend(apklist[0]['antiFeatures'])
+            antiFeatures.extend(apklist[0]['antiFeatures'])
-        if app.AntiFeatures:
+        if app.get("NoSourceSince"):
-            afout = sorted(set(app.AntiFeatures))
+            antiFeatures.append("NoSourceSince")
+        if antiFeatures:
+            afout = sorted(set(antiFeatures))
             addElementNonEmpty('antifeatures', ','.join(afout), doc, apel)
 
         # Check for duplicates - they will make the client unhappy...

fdroidserver/update.py

@@ -1821,9 +1821,6 @@ def apply_info_from_latest_apk(apps, apks):
                     bestver = apk['versionCode']
                     bestapk = apk
 
-                if app.get('NoSourceSince'):
-                    apk['antiFeatures'].add('NoSourceSince')
-
         if not app['added']:
             logging.debug("Don't know when " + appid + " was added")
         if not app['lastUpdated']:

tests/index.TestCase

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import copy
 import datetime
 import inspect
 import logging
@@ -425,6 +426,7 @@ class IndexTest(unittest.TestCase):
         app['icon'] = 'info.zwanenburg.caffeinetile.4.xml'
         app['CurrentVersionCode'] = 4
         apps = {app.id: app}
+        orig_apps = copy.deepcopy(apps)
         apk = {
             'hash': 'dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db',
             'hashType': 'sha256',
@@ -436,7 +438,7 @@ class IndexTest(unittest.TestCase):
                 '-1': 'res/drawable/ic_coffee_on.xml',
             },
             'icons': {'160': 'info.zwanenburg.caffeinetile.4.xml'},
-            'antiFeatures': [],
+            'antiFeatures': ['KnownVuln'],
             'packageName': 'info.zwanenburg.caffeinetile',
             'versionCode': 4,
             'name': 'Caffeine Tile',
@@ -463,6 +465,7 @@ class IndexTest(unittest.TestCase):
             )
         )
         self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
+        self.assertEqual(orig_apps, apps, "apps was modified when building the index")
 
     def test_v0_invalid_config_exception(self):
         """Index v0 needs additional config values when using --nosign
@@ -663,7 +666,7 @@ if __name__ == "__main__":
         default=False,
         help="Spew out even more information than normal",
     )
-    (options, args) = parser.parse_args()
+    (options, args) = parser.parse_args(["--verbose"])
     Options.verbose = options.verbose
 
     newSuite = unittest.TestSuite()

tests/metadata/com.politedroid.yml

@@ -1,3 +1,5 @@
+AntiFeatures:
+  - NonFreeNet
 Categories:
   - Time
 License: GPL-3.0-only

tests/metadata/dump/com.politedroid.yaml

@@ -1,5 +1,5 @@
 AllowedAPKSigningKeys: []
-AntiFeatures: []
+AntiFeatures: ['NonFreeNet']
 ArchivePolicy: 4 versions
 AuthorEmail: null
 AuthorName: null

tests/repo/entry.json

@@ -3,9 +3,9 @@
   "version": 20002,
   "index": {
     "name": "/index-v2.json",
-    "sha256": "da1d651eb7bbc27d2334819c591baa5afb70b01397849de515dade624a96de6d",
+    "sha256": "2f92210a7c7f2f3d855006979ebce4dda203de5ae6596a783aa531f8086e2694",
     "size": 32946,
     "numPackages": 10
   },
   "diffs": {}
-}
+}

tests/repo/index-v1.json

@@ -170,7 +170,8 @@
     },
     {
       "antiFeatures": [
-        "NoSourceSince"
+        "NoSourceSince",
+        "NonFreeNet"
       ],
       "categories": [
         "Time"
@@ -230,9 +231,6 @@
     "com.politedroid": [
       {
         "added": 1498176000000,
-        "antiFeatures": [
-          "NoSourceSince"
-        ],
         "apkName": "com.politedroid_6.apk",
         "hash": "70c2f776a2bac38a58a7d521f96ee0414c6f0fb1de973c3ca8b10862a009247d",
         "hashType": "sha256",
@@ -257,9 +255,6 @@
       },
       {
         "added": 1498176000000,
-        "antiFeatures": [
-          "NoSourceSince"
-        ],
         "apkName": "com.politedroid_5.apk",
         "hash": "5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d",
         "hashType": "sha256",
@@ -284,9 +279,6 @@
       },
       {
         "added": 1498176000000,
-        "antiFeatures": [
-          "NoSourceSince"
-        ],
         "apkName": "com.politedroid_4.apk",
         "hash": "c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075",
         "hashType": "sha256",
@@ -324,7 +316,6 @@
         "added": 1498176000000,
         "antiFeatures": [
           "KnownVuln",
-          "NoSourceSince",
           "NonFreeAssets",
           "UpstreamNonFree"
         ],
@@ -698,4 +689,4 @@
       }
     ]
   }
-}
+}

tests/repo/index-v2.json

@@ -72,8 +72,7 @@
           "file": {
             "name": "/com.politedroid_6.apk",
             "sha256": "70c2f776a2bac38a58a7d521f96ee0414c6f0fb1de973c3ca8b10862a009247d",
-            "size": 16578,
+            "size": 16578
-            "ipfsCIDv1": "bafybeidvgxrq77qr7yqkcnykdfvszsxjqc5kzt6ya5k7r666wriadrylt4"
           },
           "manifest": {
             "versionName": "1.5",
@@ -97,7 +96,8 @@
             ]
           },
           "antiFeatures": {
-            "NoSourceSince": {}
+            "NoSourceSince": {},
+            "NonFreeNet": {}
           }
         },
         "5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d": {
@@ -105,8 +105,7 @@
           "file": {
             "name": "/com.politedroid_5.apk",
             "sha256": "5bdbfa071cca4b8d05ced41d6b28763595d6e8096cca5bbf0f9253c9a2622e5d",
-            "size": 18817,
+            "size": 18817
-            "ipfsCIDv1": "bafybeifbrio5rumqvgfd5sihs7yihux2yktfvd5i7jimlgrwchzcvi6ldu"
           },
           "manifest": {
             "versionName": "1.4",
@@ -130,7 +129,8 @@
             ]
           },
           "antiFeatures": {
-            "NoSourceSince": {}
+            "NoSourceSince": {},
+            "NonFreeNet": {}
           }
         },
         "c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075": {
@@ -138,8 +138,7 @@
           "file": {
             "name": "/com.politedroid_4.apk",
             "sha256": "c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075",
-            "size": 18489,
+            "size": 18489
-            "ipfsCIDv1": "bafybeicridbev22c2rt3lwbfsrkafcf3yepak7kpvk6zgbayrxls2mmwim"
           },
           "manifest": {
             "versionName": "1.3",
@@ -172,7 +171,8 @@
             ]
           },
           "antiFeatures": {
-            "NoSourceSince": {}
+            "NoSourceSince": {},
+            "NonFreeNet": {}
           }
         },
         "665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d": {
@@ -180,8 +180,7 @@
           "file": {
             "name": "/com.politedroid_3.apk",
             "sha256": "665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d",
-            "size": 17552,
+            "size": 17552
-            "ipfsCIDv1": "bafybeib7arokhivttalcnq5ieu5fx5pzn7vo5qpmdiozqodzhb4ba53nd4"
           },
           "manifest": {
             "versionName": "1.2",
@@ -214,7 +213,11 @@
             ]
           },
           "antiFeatures": {
-            "NoSourceSince": {}
+            "KnownVuln": {},
+            "NoSourceSince": {},
+            "NonFreeAssets": {},
+            "NonFreeNet": {},
+            "UpstreamNonFree": {}
           }
         }
       }
@@ -247,8 +250,7 @@
           "file": {
             "name": "/duplicate.permisssions_9999999.apk",
             "sha256": "8367857fe75f85321ce2c344b34804d0bc193707f6ba03710d025d9030803434",
-            "size": 27446,
+            "size": 27446
-            "ipfsCIDv1": "bafybeicucr4lk7fynyde4fpxubudpl6m6wqnuq2j6vjroutjyryw24en3u"
           },
           "manifest": {
             "versionName": "",
@@ -390,8 +392,7 @@
           "file": {
             "name": "/urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk",
             "sha256": "15c0ec72c74a3791f42cdb43c57df0fb11a4dbb656851bbb8cf05b26a8372789",
-            "size": 11471,
+            "size": 11471
-            "ipfsCIDv1": "bafybeig77jwqx243si3gh55iqx4gkcxhltkt6pjimzgigfsk3kshsi6qem"
           },
           "manifest": {
             "versionName": "0.1",
@@ -430,8 +431,7 @@
           "file": {
             "name": "/info.zwanenburg.caffeinetile_4.apk",
             "sha256": "dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db",
-            "size": 11740,
+            "size": 11740
-            "ipfsCIDv1": "bafybeigormhkorw3mk6pkkfk63kkmxpvwylthgj67geulvskc5acr65sym"
           },
           "manifest": {
             "versionName": "1.3",
@@ -482,8 +482,7 @@
           "file": {
             "name": "/no.min.target.sdk_987.apk",
             "sha256": "e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c",
-            "size": 14102,
+            "size": 14102
-            "ipfsCIDv1": "bafybeidwxseoagnew3gtlasttqovl7ciuwxaud5a5p4a5pzpbrfcfj2gaa"
           },
           "manifest": {
             "versionName": "1.2-fake",
@@ -541,8 +540,7 @@
           "file": {
             "name": "/obb.main.oldversion_1444412523.apk",
             "sha256": "c5f149e526f89c05c62923bdb7bb1e2be5673c46ec85143f41e514340631449c",
-            "size": 14323,
+            "size": 14323
-            "ipfsCIDv1": "bafybeicnwnpiyfke3tbk3nve62meig65vved34i6kesjkksdciff6242ui"
           },
           "obbMainFile": {
             "name": "/main.1434483388.obb.main.oldversion.obb",
@@ -639,8 +637,7 @@
           "file": {
             "name": "/obb.main.twoversions_1101617.apk",
             "sha256": "9bc74566f089ef030ac33e7fbd99d92f1a38f363fb499fed138d9e7b774e821c",
-            "size": 11481,
+            "size": 11481
-            "ipfsCIDv1": "bafybeiblpfmwololxgsrum337rbbbsqg2gk6hytvt6szf4njubosju3bme"
           },
           "src": {
             "name": "/obb.main.twoversions_1101617_src.tar.gz",
@@ -671,8 +668,7 @@
           "file": {
             "name": "/obb.main.twoversions_1101615.apk",
             "sha256": "7b0b7b9ba248e15751a16e3a0e01e1e24cbb673686c38422030cb75d5c33f0bb",
-            "size": 11480,
+            "size": 11480
-            "ipfsCIDv1": "bafybeigglr3iefb3es4lp2sgfacppk3w2qqtuykjgf4actebpalyizef3q"
           },
           "obbMainFile": {
             "name": "/main.1101615.obb.main.twoversions.obb",
@@ -698,8 +694,7 @@
           "file": {
             "name": "/obb.main.twoversions_1101613.apk",
             "sha256": "cce97a52ff18d843185be7f22ecb1a557c36b7a9f8ba07a8be94e328e00b35dc",
-            "size": 11477,
+            "size": 11477
-            "ipfsCIDv1": "bafybeicocjo4khzp2rkui2ltvrhbksrm373lr3pb43ut7hqgbllfjpv6ti"
           },
           "obbMainFile": {
             "name": "/main.1101613.obb.main.twoversions.obb",
@@ -777,8 +772,7 @@
           "file": {
             "name": "/obb.mainpatch.current_1619.apk",
             "sha256": "eda5fc3ecfdac3252717e36bdbc9820865baeef162264af9ba5db7364f0e7a0c",
-            "size": 11479,
+            "size": 11479
-            "ipfsCIDv1": "bafybeievo4e234mllujityvtjgeltauyfbriszoqddzygmimcm4mo3zyqu"
           },
           "obbMainFile": {
             "name": "/main.1619.obb.mainpatch.current.obb",
@@ -809,8 +803,7 @@
           "file": {
             "name": "/obb.mainpatch.current_1619_another-release-key.apk",
             "sha256": "42e7d6d2f8254aaf9fe95ba6ecc233ee8c3cd543a3e4f3f9ebe1b638221122fa",
-            "size": 10541,
+            "size": 10541
-            "ipfsCIDv1": "bafybeiatdbzlxairqzvdowevwuy7nk24rknc55jpip2wb2sq4c3f7mtngm"
           },
           "obbMainFile": {
             "name": "/main.1619.obb.mainpatch.current.obb",
@@ -877,8 +870,7 @@
           "file": {
             "name": "/souch.smsbypass_9.apk",
             "sha256": "80b0ae68a1189baa3ee6717092e3dbf1a4210165f7f7e5f2f9616bd63a2ec01d",
-            "size": 81295,
+            "size": 81295
-            "ipfsCIDv1": "bafybeihaccfnt32q2iwfulh2m7jvdivuunlw6t72wa7jfi7igxvqxjqszy"
           },
           "manifest": {
             "versionName": "0.9",
@@ -917,4 +909,4 @@
       }
     }
   }
-}
+}

tests/repo/index.xml

@@ -318,7 +318,7 @@ APK is called F-Droid Privileged Extension.</desc>
 		<tracker>https://github.com/miguelvps/PoliteDroid/issues</tracker>
 		<marketversion>1.5</marketversion>
 		<marketvercode>6</marketvercode>
-		<antifeatures>NoSourceSince</antifeatures>
+		<antifeatures>NoSourceSince,NonFreeNet</antifeatures>
 		<package>
 			<version>1.5</version>
 			<versioncode>6</versioncode>

tests/run-tests

@@ -306,6 +306,8 @@ $sed -i.tmp -e 's,timestamp="[0-9]*",timestamp="1676634233",' repo/index.xml
 diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml
 sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v1.json
 diff -uw $WORKSPACE/tests/repo/index-v1.json repo/index-v1.json
+sed -i --expression='s,"timestamp": [0-9]*,"timestamp": 1676634233000,' repo/index-v2.json
+diff -uw $WORKSPACE/tests/repo/index-v2.json repo/index-v2.json
 
 
 #------------------------------------------------------------------------------#