From 1c3a87e002341ee74782e9cdd4031ade5178cd15 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 30 May 2023 19:19:27 +0200 Subject: [PATCH 1/3] lint: get Categories from config --- fdroidserver/lint.py | 40 +++++++++++++------------------- tests/lint.TestCase | 55 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 24 deletions(-) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 0fc4aa5e..b8c85f6d 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -222,6 +222,7 @@ versioncode_check_pattern = re.compile(r"(\\d|\[(0-9|\\d)_?(a-fA-F)?])[+]") ANTIFEATURES_KEYS = None ANTIFEATURES_PATTERN = None +CATEGORIES_KEYS = list() def load_antiFeatures_config(): @@ -234,6 +235,18 @@ def load_antiFeatures_config(): ANTIFEATURES_PATTERN = ','.join(ANTIFEATURES_KEYS) +def load_categories_config(): + """Lazy loading, since it might read a lot of files.""" + global CATEGORIES_KEYS + k = 'categories' + if not CATEGORIES_KEYS: + if config and k in config: + CATEGORIES_KEYS = config[k] + else: + config[k] = common.load_localized_config(k, 'repo') + CATEGORIES_KEYS = list(config[k].keys()) + + def check_regexes(app): for f, checks in regex_checks.items(): for m, r in checks: @@ -371,32 +384,10 @@ def check_empty_fields(app): yield _("Categories are not set") -all_categories = set( - [ - "Connectivity", - "Development", - "Games", - "Graphics", - "Internet", - "Money", - "Multimedia", - "Navigation", - "Phone & SMS", - "Reading", - "Science & Education", - "Security", - "Sports & Health", - "System", - "Theming", - "Time", - "Writing", - ] -) - - def check_categories(app): + """App uses 'Categories' key and parsed config uses 'categories' key.""" for categ in app.Categories: - if categ not in all_categories: + if categ not in CATEGORIES_KEYS: yield _("Categories '%s' is not valid" % categ) @@ -798,6 +789,7 @@ def main(): config = common.read_config(options) load_antiFeatures_config() + load_categories_config() # Get all apps... allapps = metadata.read_metadata(options.appid) diff --git a/tests/lint.TestCase b/tests/lint.TestCase index 544b18c8..af4fe8f1 100755 --- a/tests/lint.TestCase +++ b/tests/lint.TestCase @@ -10,6 +10,7 @@ import sys import tempfile import unittest from pathlib import Path +from testcommon import mkdtemp localmodule = Path(__file__).resolve().parent.parent print('localmodule: ' + str(localmodule)) @@ -30,6 +31,14 @@ class LintTest(unittest.TestCase): self.tmpdir = localmodule / '.testfiles' self.tmpdir.mkdir(exist_ok=True) os.chdir(self.basedir) + fdroidserver.common.config = None + fdroidserver.lint.config = None + fdroidserver.lint.CATEGORIES_KEYS = None + self._td = mkdtemp() + self.testdir = self._td.name + + def tearDown(self): + self._td.cleanup() def test_check_for_unsupported_metadata_files(self): self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files()) @@ -313,12 +322,58 @@ class LintTest(unittest.TestCase): logging.debug(warn) self.assertFalse(anywarns) + def test_check_categories_in_config(self): + fdroidserver.lint.config = {'categories': ['InConfig']} + fdroidserver.lint.load_categories_config() + app = fdroidserver.metadata.App({'Categories': ['InConfig']}) + self.assertEqual(0, len(list(fdroidserver.lint.check_categories(app)))) + + def test_check_categories_not_in_config(self): + fdroidserver.lint.config = dict() + fdroidserver.lint.load_categories_config() + app = fdroidserver.metadata.App({'Categories': ['NotInConfig']}) + self.assertEqual(1, len(list(fdroidserver.lint.check_categories(app)))) + + def test_check_categories_empty_is_error(self): + fdroidserver.lint.config = {'categories': []} + fdroidserver.lint.load_categories_config() + app = fdroidserver.metadata.App({'Categories': ['something']}) + self.assertEqual(1, len(list(fdroidserver.lint.check_categories(app)))) + + def test_check_categories_old_hardcoded_not_defined(self): + fdroidserver.lint.config = {'categories': ['foo', 'bar']} + fdroidserver.lint.load_categories_config() + app = fdroidserver.metadata.App({'Categories': ['Writing']}) + self.assertEqual(1, len(list(fdroidserver.lint.check_categories(app)))) + + def test_check_categories_from_config_yml(self): + """In config.yml, categories is a list.""" + os.chdir(self.testdir) + Path('config.yml').write_text('categories: [foo, bar]') + fdroidserver.lint.config = fdroidserver.common.read_config() + fdroidserver.lint.load_categories_config() + self.assertEqual(fdroidserver.lint.CATEGORIES_KEYS, ['foo', 'bar']) + app = fdroidserver.metadata.App({'Categories': ['bar']}) + self.assertEqual(0, len(list(fdroidserver.lint.check_categories(app)))) + + def test_check_categories_from_config_categories_yml(self): + """In config/categories.yml, categories is a localized STRINGMAP dict.""" + os.chdir(self.testdir) + os.mkdir('config') + Path('config/categories.yml').write_text('{foo: {name: foo}, bar: {name: bar}}') + fdroidserver.lint.config = fdroidserver.common.read_config() + fdroidserver.lint.load_categories_config() + self.assertEqual(fdroidserver.lint.CATEGORIES_KEYS, ['foo', 'bar']) + app = fdroidserver.metadata.App({'Categories': ['bar']}) + self.assertEqual(0, len(list(fdroidserver.lint.check_categories(app)))) + class LintAntiFeaturesTest(unittest.TestCase): def setUp(self): self.basedir = localmodule / 'tests' os.chdir(self.basedir) fdroidserver.common.config = dict() + fdroidserver.lint.ANTIFEATURES_KEYS = None fdroidserver.lint.load_antiFeatures_config() def test_check_antiFeatures_empty(self): From bc589593159d3f3e014e0deaa4956ac0cdaaee3d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 30 May 2023 22:42:22 +0200 Subject: [PATCH 2/3] purge remnants of repo/categories.txt, which is long unused --- MANIFEST.in | 1 - fdroidserver/common.py | 1 - fdroidserver/update.py | 15 --------------- tests/common.TestCase | 7 +------ tests/repo/categories.txt | 10 ---------- tests/update.TestCase | 15 +++++++++++++++ 6 files changed, 16 insertions(+), 33 deletions(-) delete mode 100644 tests/repo/categories.txt diff --git a/MANIFEST.in b/MANIFEST.in index e301d8a8..ea144609 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -662,7 +662,6 @@ include tests/org.bitbucket.tickytacky.mirrormirror_4.apk include tests/org.dyndns.fules.ck_20.apk include tests/org.sajeg.fallingblocks_3.apk include tests/publish.TestCase -include tests/repo/categories.txt include tests/repo/com.example.test.helloworld_1.apk include tests/repo/com.politedroid_3.apk include tests/repo/com.politedroid_4.apk diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 1af0fc82..73f25365 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4042,7 +4042,6 @@ def is_repo_file(filename, for_gpg_signing=False): if isinstance(filename, str): filename = filename.encode('utf-8', errors="surrogateescape") ignore_files = [ - b'categories.txt', b'entry.jar', b'index-v1.jar', b'index.css', diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 86070c67..44e95c28 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1846,15 +1846,6 @@ def apply_info_from_latest_apk(apps, apks): app['CurrentVersionCode'] = bestver -def make_categories_txt(repodir, categories): - """Write a category list in the repo to allow quick access.""" - catdata = '' - for cat in sorted(categories): - catdata += cat + '\n' - with open(os.path.join(repodir, 'categories.txt'), 'w') as f: - f.write(catdata) - - def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions): def filter_apk_list_sorted(apk_list): apkList = [] @@ -2246,11 +2237,6 @@ def main(): # Get all apps... apps = metadata.read_metadata() - # Generate a list of categories... - categories = set() - for app in apps.values(): - categories.update(app.Categories) - # Read known apks data (will be updated and written back when we've finished) knownapks = common.KnownApks() @@ -2363,7 +2349,6 @@ def main(): # Make the index for the main repo... index.make(repoapps, apks, repodirs[0], False) - make_categories_txt(repodirs[0], categories) git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): diff --git a/tests/common.TestCase b/tests/common.TestCase index 8820004d..56625598 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -2509,8 +2509,7 @@ class CommonTest(unittest.TestCase): 'repo/index.png', 'repo/index.xml', ] - non_repo_files = ['repo/categories.txt'] - for f in repo_files + index_files + non_repo_files: + for f in repo_files + index_files: open(f, 'w').close() repo_dirs = [ @@ -2543,10 +2542,6 @@ class CommonTest(unittest.TestCase): self.assertTrue(os.path.exists(d), d + ' was created') self.assertFalse(is_repo_file(d), d + ' not repo file') - for f in non_repo_files: - self.assertTrue(os.path.exists(f), f + ' was created') - self.assertFalse(is_repo_file(f), f + ' not repo file') - def test_get_apksigner_smartcardoptions(self): os.chdir(self.tmpdir) with open('config.yml', 'w') as fp: diff --git a/tests/repo/categories.txt b/tests/repo/categories.txt deleted file mode 100644 index 9af231f5..00000000 --- a/tests/repo/categories.txt +++ /dev/null @@ -1,10 +0,0 @@ -1 -2.0 -Development -GuardianProject -Multimedia -Phone & SMS -Security -System -Time -tests diff --git a/tests/update.TestCase b/tests/update.TestCase index 2552ce0b..391dec93 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -22,6 +22,7 @@ import zipfile import textwrap from datetime import datetime from distutils.version import LooseVersion +from pathlib import Path from testcommon import TmpCwd, mkdtemp from unittest import mock @@ -1789,6 +1790,20 @@ class UpdateTest(unittest.TestCase): "DEBUG:root:Checking archiving for org.smssecure.smssecure - apks:0, keepversions:6, archapks:0" ]) + def test_categories_txt_is_removed_by_delete_unknown(self): + """categories.txt used to be a part of this system, now its nothing.""" + os.chdir(self.testdir) + Path('config.yml').write_text('repo_pubkey: ffffffffffffffffffffffffffffffffffffffff') + + categories_txt = Path('repo/categories.txt') + categories_txt.parent.mkdir() + categories_txt.write_text('placeholder') + + self.assertTrue(categories_txt.exists()) + with mock.patch('sys.argv', ['fdroid update', '--delete-unknown', '--nosign']): + fdroidserver.update.main() + self.assertFalse(categories_txt.exists()) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From 6b827100394ef6df4a85d138151e6741655d9663 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 30 May 2023 22:43:21 +0200 Subject: [PATCH 3/3] test load_localized_config() with categories --- MANIFEST.in | 1 + tests/common.TestCase | 17 +++++++++++++++++ tests/config/categories.yml | 14 ++++++++++++++ tests/repo/entry.json | 4 ++-- tests/repo/index-v2.json | 37 +++++++++++++++++++++++++++++++++++++ tests/run-tests | 6 +++++- 6 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 tests/config/categories.yml diff --git a/MANIFEST.in b/MANIFEST.in index ea144609..1aed9975 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -546,6 +546,7 @@ include tests/checkupdates.TestCase include tests/common.TestCase include tests/config.py include tests/config/antiFeatures.yml +include tests/config/categories.yml include tests/config/de/antiFeatures.yml include tests/config/fa/antiFeatures.yml include tests/config/ic_antifeature_ads.xml diff --git a/tests/common.TestCase b/tests/common.TestCase index 56625598..9bda9820 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -2694,6 +2694,23 @@ class CommonTest(unittest.TestCase): p = Path(os.path.dirname(__file__) + '/repo' + v['icon']['en-US']['name']) self.assertTrue(p.exists()) + def test_load_localized_config_categories(self): + """It should load""" + categories = fdroidserver.common.load_localized_config('categories', 'repo') + self.assertEqual( + [ + 'Time', + 'Development', + 'GuardianProject', + 'Multimedia', + 'Phone & SMS', + 'Security', + 'System', + ], + list(categories.keys()), + ) + self.assertEqual(['en-US'], list(categories['GuardianProject']['name'].keys())) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/config/categories.yml b/tests/config/categories.yml new file mode 100644 index 00000000..e294ced6 --- /dev/null +++ b/tests/config/categories.yml @@ -0,0 +1,14 @@ +Time: + name: Time +Development: + name: Development +GuardianProject: + name: Guardian Project +Multimedia: + name: Multimedia +Phone & SMS: + name: Phone & SMS +Security: + name: Security +System: + name: System diff --git a/tests/repo/entry.json b/tests/repo/entry.json index 2f98dbab..5c0ca528 100644 --- a/tests/repo/entry.json +++ b/tests/repo/entry.json @@ -3,8 +3,8 @@ "version": 20002, "index": { "name": "/index-v2.json", - "sha256": "ba000a3f5e1935d338f374c50cae529b8ce6d988ab3ed67c7a8cf437502f81ad", - "size": 52481, + "sha256": "7117ee6ff4ff2dd71ec3f3d3ad2ef7e9fd4afead9b1f2d39d0b224a1812e78b5", + "size": 53233, "numPackages": 10 }, "diffs": {} diff --git a/tests/repo/index-v2.json b/tests/repo/index-v2.json index 5a8deb1e..6ea92407 100644 --- a/tests/repo/index-v2.json +++ b/tests/repo/index-v2.json @@ -498,6 +498,43 @@ } } }, + "categories": { + "Time": { + "name": { + "en-US": "Time" + } + }, + "Development": { + "name": { + "en-US": "Development" + } + }, + "GuardianProject": { + "name": { + "en-US": "Guardian Project" + } + }, + "Multimedia": { + "name": { + "en-US": "Multimedia" + } + }, + "Phone & SMS": { + "name": { + "en-US": "Phone & SMS" + } + }, + "Security": { + "name": { + "en-US": "Security" + } + }, + "System": { + "name": { + "en-US": "System" + } + } + }, "requests": { "install": [ "org.adaway" diff --git a/tests/run-tests b/tests/run-tests index 49acb5aa..8acb1889 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -702,7 +702,11 @@ echo "Description: |" >> metadata/fake.yml echo " this is fake" >> metadata/fake.yml # fake that no JDKs are available -echo 'java_paths: {}' > config.yml +cat > config.yml <