Merge branch 'read_metadata' into 'master'

Draft: Read only metadata of apps given on the command line

See merge request fdroid/fdroidserver!1037

examples/fdroid_fetchsrclibs.py

@@ -18,10 +18,9 @@ def main():
     common.setup_global_opts(parser)
     parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
     metadata.add_metadata_arguments(parser)
-    options = common.parse_args(parser)
+    options = parser.parse_args()
-    pkgs = common.read_pkg_args(options.appid, True)
+    common.options = options
-    allapps = metadata.read_metadata(pkgs)
+    apps = common.read_app_args(options.appid, allow_vercodes=True)
-    apps = common.read_app_args(options.appid, allapps, True)
     common.read_config()
     srclib_dir = os.path.join('build', 'srclib')
     os.makedirs(srclib_dir, exist_ok=True)

fdroidserver/build.py

@@ -1121,10 +1121,7 @@ def main():
     srclib_dir = os.path.join(build_dir, 'srclib')
     extlib_dir = os.path.join(build_dir, 'extlib')
 
-    # Read all app and srclib metadata
+    apps = common.read_app_args(options.appid, allow_vercodes=True, sort_by_time=True)
-    pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(pkgs, sort_by_time=True)
-    apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
         if (app.get('Disabled') and not options.force) or not app.get('RepoType') or not app.get('Builds', []):

fdroidserver/checkupdates.py

@@ -730,10 +730,7 @@ def main():
             logging.error(_('Build metadata git repo has uncommited changes!'))
             sys.exit(1)
 
-    # Get all apps...
+    apps = common.read_app_args(options.appid)
-    allapps = metadata.read_metadata()
-
-    apps = common.read_app_args(options.appid, allapps, False)
 
     processed = []
     failed = dict()

fdroidserver/common.py

@@ -932,7 +932,7 @@ def get_metadata_files(vercodes):
     return metadatafiles
 
 
-def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
+def read_app_args(appid_versionCode_pairs, allow_vercodes=False, sort_by_time=False):
     """Build a list of App instances for processing.
 
     On top of what read_pkg_args does, this returns the whole app
@@ -943,6 +943,7 @@ def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
 
     """
     vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
+    allapps = fdroidserver.metadata.read_metadata(vercodes, sort_by_time)
 
     if not vercodes:
         return allapps
@@ -952,11 +953,6 @@ def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
         if appid in vercodes:
             apps[appid] = app
 
-    if len(apps) != len(vercodes):
-        for p in vercodes:
-            if p not in allapps:
-                logging.critical(_("No such package: %s") % p)
-        raise FDroidException(_("Found invalid appids in arguments"))
     if not apps:
         raise FDroidException(_("No packages specified"))
 

fdroidserver/lint.py

@@ -850,9 +850,7 @@ def main():
 
 
 def lint_metadata(options):
-    # Get all apps...
+    apps = common.read_app_args(options.appid)
-    allapps = metadata.read_metadata(options.appid)
-    apps = common.read_app_args(options.appid, allapps, False)
 
     anywarns = check_for_unsupported_metadata_files()
 

fdroidserver/metadata.py

@@ -551,7 +551,7 @@ def read_srclibs():
         srclibs[metadatapath.stem] = parse_yaml_srclib(metadatapath)
 
 
-def read_metadata(appids={}, sort_by_time=False):
+def read_metadata(appid_to_vercode={}, sort_by_time=False):
     """Return a list of App instances sorted newest first.
 
     This reads all of the metadata files in a 'data' repository, then
@@ -559,7 +559,7 @@ def read_metadata(appids={}, sort_by_time=False):
     sorted based on creation time, newest first.  Most of the time,
     the newer files are the most interesting.
 
-    appids is a dict with appids a keys and versionCodes as values.
+    appid_to_vercode is a dict with appids a keys and versionCodes as values.
 
     """
     # Always read the srclibs before the apps, since they can use a srlib as
@@ -571,9 +571,8 @@ def read_metadata(appids={}, sort_by_time=False):
     for basedir in ('metadata', 'tmp'):
         Path(basedir).mkdir(exist_ok=True)
 
-    if appids:
+    if appid_to_vercode:
-        vercodes = common.read_pkg_args(appids)
+        metadatafiles = common.get_metadata_files(appid_to_vercode)
-        metadatafiles = common.get_metadata_files(vercodes)
     else:
         metadatafiles = list(Path('metadata').glob('*.yml')) + list(
             Path('.').glob('.fdroid.yml')

fdroidserver/rewritemeta.py

@@ -81,9 +81,7 @@ def main():
 
     config = common.read_config()
 
-    # Get all apps...
+    apps = common.read_app_args(options.appid)
-    allapps = metadata.read_metadata(options.appid)
-    apps = common.read_app_args(options.appid, allapps, False)
 
     for appid, app in apps.items():
         path = Path(app.metadatapath)

fdroidserver/scanner.py

@@ -827,10 +827,7 @@ def main():
             _get_tool()
         return
 
-    # Read all app and srclib metadata
+    apps = common.read_app_args(appids, allow_vercodes=True)
-
-    allapps = metadata.read_metadata()
-    apps = common.read_app_args(appids, allapps, True)
 
     build_dir = 'build'
     if not os.path.isdir(build_dir):

tests/scanner.TestCase

@@ -13,7 +13,6 @@ import unittest
 import uuid
 import yaml
 import zipfile
-import collections
 import pathlib
 from unittest import mock
 from dataclasses import asdict
@@ -786,7 +785,7 @@ class Test_main(unittest.TestCase):
 
             self.exit_func.assert_not_called()
             self.read_app_args_func.assert_called_once_with(
-                ['com.example.app'], collections.OrderedDict(), True
+                ['com.example.app'], allow_vercodes=True
             )
             self.scan_binary_func.assert_not_called()