From 57c25d5dd29c4eeba6d47ce8409c355ce212e15c Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 19 Jul 2022 08:48:47 +0200 Subject: [PATCH 1/3] Read only metadata of apps given on the command line --- examples/fdroid_fetchsrclibs.py | 7 +++---- fdroidserver/build.py | 5 +---- fdroidserver/checkupdates.py | 5 +---- fdroidserver/common.py | 3 ++- fdroidserver/lint.py | 4 +--- fdroidserver/rewritemeta.py | 4 +--- fdroidserver/scanner.py | 5 +---- tests/scanner.TestCase | 3 +-- 8 files changed, 11 insertions(+), 25 deletions(-) diff --git a/examples/fdroid_fetchsrclibs.py b/examples/fdroid_fetchsrclibs.py index 299e802b..71a8c457 100644 --- a/examples/fdroid_fetchsrclibs.py +++ b/examples/fdroid_fetchsrclibs.py @@ -18,10 +18,9 @@ def main(): common.setup_global_opts(parser) parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]")) metadata.add_metadata_arguments(parser) - options = common.parse_args(parser) - pkgs = common.read_pkg_args(options.appid, True) - allapps = metadata.read_metadata(pkgs) - apps = common.read_app_args(options.appid, allapps, True) + options = parser.parse_args() + common.options = options + apps = common.read_app_args(options.appid, allow_vercodes=True) common.read_config() srclib_dir = os.path.join('build', 'srclib') os.makedirs(srclib_dir, exist_ok=True) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index a1fc2ff3..bb48a2e4 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1121,10 +1121,7 @@ def main(): srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') - # Read all app and srclib metadata - pkgs = common.read_pkg_args(options.appid, True) - allapps = metadata.read_metadata(pkgs, sort_by_time=True) - apps = common.read_app_args(options.appid, allapps, True) + apps = common.read_app_args(options.appid, allow_vercodes=True, sort_by_time=True) for appid, app in list(apps.items()): if (app.get('Disabled') and not options.force) or not app.get('RepoType') or not app.get('Builds', []): diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 358a6e70..670cb6cb 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -730,10 +730,7 @@ def main(): logging.error(_('Build metadata git repo has uncommited changes!')) sys.exit(1) - # Get all apps... - allapps = metadata.read_metadata() - - apps = common.read_app_args(options.appid, allapps, False) + apps = common.read_app_args(options.appid) processed = [] failed = dict() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5a546386..61e19e82 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -932,7 +932,7 @@ def get_metadata_files(vercodes): return metadatafiles -def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False): +def read_app_args(appid_versionCode_pairs, allow_vercodes=False, sort_by_time=False): """Build a list of App instances for processing. On top of what read_pkg_args does, this returns the whole app @@ -943,6 +943,7 @@ def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False): """ vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes) + allapps = fdroidserver.metadata.read_metadata(appid_versionCode_pairs, sort_by_time) if not vercodes: return allapps diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 1ae42852..a38ad0eb 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -850,9 +850,7 @@ def main(): def lint_metadata(options): - # Get all apps... - allapps = metadata.read_metadata(options.appid) - apps = common.read_app_args(options.appid, allapps, False) + apps = common.read_app_args(options.appid) anywarns = check_for_unsupported_metadata_files() diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 7f003ac3..9f3316b4 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -81,9 +81,7 @@ def main(): config = common.read_config() - # Get all apps... - allapps = metadata.read_metadata(options.appid) - apps = common.read_app_args(options.appid, allapps, False) + apps = common.read_app_args(options.appid) for appid, app in apps.items(): path = Path(app.metadatapath) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index fc9ef13a..b0f58fcd 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -827,10 +827,7 @@ def main(): _get_tool() return - # Read all app and srclib metadata - - allapps = metadata.read_metadata() - apps = common.read_app_args(appids, allapps, True) + apps = common.read_app_args(appids, allow_vercodes=True) build_dir = 'build' if not os.path.isdir(build_dir): diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index c412b32e..2028be8a 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -13,7 +13,6 @@ import unittest import uuid import yaml import zipfile -import collections import pathlib from unittest import mock from dataclasses import asdict @@ -786,7 +785,7 @@ class Test_main(unittest.TestCase): self.exit_func.assert_not_called() self.read_app_args_func.assert_called_once_with( - ['com.example.app'], collections.OrderedDict(), True + ['com.example.app'], allow_vercodes=True ) self.scan_binary_func.assert_not_called() From bdceb178efd99456f4f89e58abc5576b8829bf58 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 19 Jul 2022 13:41:24 +0200 Subject: [PATCH 2/3] Use vercodes in read_metadata interface read_metadata is only called in read_app_args with an argument. As vercodes is already there, drop the duplicate call. --- fdroidserver/common.py | 2 +- fdroidserver/metadata.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 61e19e82..302fab6f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -943,7 +943,7 @@ def read_app_args(appid_versionCode_pairs, allow_vercodes=False, sort_by_time=Fa """ vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes) - allapps = fdroidserver.metadata.read_metadata(appid_versionCode_pairs, sort_by_time) + allapps = fdroidserver.metadata.read_metadata(vercodes, sort_by_time) if not vercodes: return allapps diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ce7e5ee8..815d6b0f 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -551,7 +551,7 @@ def read_srclibs(): srclibs[metadatapath.stem] = parse_yaml_srclib(metadatapath) -def read_metadata(appids={}, sort_by_time=False): +def read_metadata(appid_to_vercode={}, sort_by_time=False): """Return a list of App instances sorted newest first. This reads all of the metadata files in a 'data' repository, then @@ -559,7 +559,7 @@ def read_metadata(appids={}, sort_by_time=False): sorted based on creation time, newest first. Most of the time, the newer files are the most interesting. - appids is a dict with appids a keys and versionCodes as values. + appid_to_vercode is a dict with appids a keys and versionCodes as values. """ # Always read the srclibs before the apps, since they can use a srlib as @@ -571,9 +571,8 @@ def read_metadata(appids={}, sort_by_time=False): for basedir in ('metadata', 'tmp'): Path(basedir).mkdir(exist_ok=True) - if appids: - vercodes = common.read_pkg_args(appids) - metadatafiles = common.get_metadata_files(vercodes) + if appid_to_vercode: + metadatafiles = common.get_metadata_files(appid_to_vercode) else: metadatafiles = list(Path('metadata').glob('*.yml')) + list( Path('.').glob('.fdroid.yml') From 5142cd8689c770bfce0f26f758e92192b730bf3a Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 19 Jul 2022 13:29:28 +0200 Subject: [PATCH 3/3] Drop duplicated code (already done in get_metadata_files) --- fdroidserver/common.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 302fab6f..78cfd7c7 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -953,11 +953,6 @@ def read_app_args(appid_versionCode_pairs, allow_vercodes=False, sort_by_time=Fa if appid in vercodes: apps[appid] = app - if len(apps) != len(vercodes): - for p in vercodes: - if p not in allapps: - logging.critical(_("No such package: %s") % p) - raise FDroidException(_("Found invalid appids in arguments")) if not apps: raise FDroidException(_("No packages specified"))