Merge branch 'read_metadata' into 'master'

Read only metadata of apps given on the command line

See merge request fdroid/fdroidserver!1037

.gitlab-ci.yml

@@ -285,9 +285,7 @@ black:
     - black --check --diff --color $CI_PROJECT_DIR
 
 fedora_latest:
-  image: fedora:latest
-  only:
-    - master@fdroid/fdroidserver
+  image: fedora:39  # support ends on 2024-11-12
   script:
     # tricks to hopefully make runs more reliable
     - echo "timeout=600" >> /etc/dnf/dnf.conf

examples/fdroid_fetchsrclibs.py

@@ -20,9 +20,7 @@ def main():
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     common.options = options
-    pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(pkgs)
-    apps = common.read_app_args(options.appid, allapps, True)
+    apps = common.read_app_args(options.appid, allow_vercodes=True)
     common.read_config(options)
     srclib_dir = os.path.join('build', 'srclib')
     os.makedirs(srclib_dir, exist_ok=True)

fdroidserver/build.py

@@ -974,10 +974,7 @@ def main():
     srclib_dir = os.path.join(build_dir, 'srclib')
     extlib_dir = os.path.join(build_dir, 'extlib')
 
-    # Read all app and srclib metadata
-    pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(pkgs, sort_by_time=True)
-    apps = common.read_app_args(options.appid, allapps, True)
+    apps = common.read_app_args(options.appid, allow_vercodes=True, sort_by_time=True)
 
     for appid, app in list(apps.items()):
         if (app.get('Disabled') and not options.force) or not app.get('RepoType') or not app.get('Builds', []):

fdroidserver/checkupdates.py

@@ -730,10 +730,7 @@ def main():
             logging.error(_('Build metadata git repo has uncommited changes!'))
             sys.exit(1)
 
-    # Get all apps...
-    allapps = metadata.read_metadata()
-
-    apps = common.read_app_args(options.appid, allapps, False)
+    apps = common.read_app_args(options.appid)
 
     processed = []
     failed = dict()

fdroidserver/common.py

@@ -891,7 +891,7 @@ def get_metadata_files(vercodes):
     return metadatafiles
 
 
-def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
+def read_app_args(appid_versionCode_pairs, allow_vercodes=False, sort_by_time=False):
     """Build a list of App instances for processing.
 
     On top of what read_pkg_args does, this returns the whole app
@@ -902,6 +902,7 @@ def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
 
     """
     vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
+    allapps = fdroidserver.metadata.read_metadata(vercodes, sort_by_time)
 
     if not vercodes:
         return allapps
@@ -911,11 +912,6 @@ def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
         if appid in vercodes:
             apps[appid] = app
 
-    if len(apps) != len(vercodes):
-        for p in vercodes:
-            if p not in allapps:
-                logging.critical(_("No such package: %s") % p)
-        raise FDroidException(_("Found invalid appids in arguments"))
     if not apps:
         raise FDroidException(_("No packages specified"))
 
@@ -2354,6 +2350,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
                 gradlefile = build_gradle
             elif os.path.exists(build_gradle_kts):
                 gradlefile = build_gradle_kts
+            else:
+                raise BuildException("No gradle file found")
             regsub_file(r'compileSdkVersion[ =]+[0-9]+',
                         r'compileSdkVersion %s' % n,
                         gradlefile)

fdroidserver/deploy.py

@@ -837,7 +837,15 @@ def push_binary_transparency(git_repo_path, git_remote):
                 origin.set_url(git_remote)
         else:
             origin = gitrepo.create_remote('origin', git_remote)
-        origin.push(GIT_BRANCH)
+        for _i in range(3):
+            try:
+                origin.push(GIT_BRANCH)
+            except git.GitCommandError as e:
+                logging.error(e)
+                continue
+            break
+        else:
+            raise FDroidException(_("Pushing to remote server failed!"))
 
 
 def main():

fdroidserver/lint.py

@@ -722,7 +722,13 @@ def check_updates_ucm_http_aum_pattern(app):  # noqa: D403
 
 
 def check_certificate_pinned_binaries(app):
-    if len(app.get('AllowedAPKSigningKeys')) > 0:
+    keys = app.get('AllowedAPKSigningKeys')
+    known_keys = common.config.get('apk_signing_key_block_list', [])
+    if keys:
+        if known_keys:
+            for key in keys:
+                if key in known_keys:
+                    yield _('Known debug key is used in AllowedAPKSigningKeys: ') + key
         return
     if app.get('Binaries') is not None:
         yield _(
@@ -842,9 +848,7 @@ def main():
 
 
 def lint_metadata(options):
-    # Get all apps...
-    allapps = metadata.read_metadata(options.appid)
-    apps = common.read_app_args(options.appid, allapps, False)
+    apps = common.read_app_args(options.appid)
 
     anywarns = check_for_unsupported_metadata_files()
 

fdroidserver/metadata.py

@@ -551,7 +551,7 @@ def read_srclibs():
         srclibs[metadatapath.stem] = parse_yaml_srclib(metadatapath)
 
 
-def read_metadata(appids={}, sort_by_time=False):
+def read_metadata(vercodes={}, sort_by_time=False):
     """Return a list of App instances sorted newest first.
 
     This reads all of the metadata files in a 'data' repository, then
@@ -559,7 +559,7 @@ def read_metadata(appids={}, sort_by_time=False):
     sorted based on creation time, newest first.  Most of the time,
     the newer files are the most interesting.
 
-    appids is a dict with appids a keys and versionCodes as values.
+    vercodes is a dict with appids a keys and versionCodes as values.
 
     """
     # Always read the srclibs before the apps, since they can use a srlib as
@@ -571,8 +571,7 @@ def read_metadata(appids={}, sort_by_time=False):
     for basedir in ('metadata', 'tmp'):
         Path(basedir).mkdir(exist_ok=True)
 
-    if appids:
-        vercodes = common.read_pkg_args(appids)
+    if vercodes:
         metadatafiles = common.get_metadata_files(vercodes)
     else:
         metadatafiles = list(Path('metadata').glob('*.yml')) + list(

fdroidserver/mirror.py

@@ -268,7 +268,8 @@ def main():
                 urls[icondir].append(url)
 
         for icondir in icondirs:
-            _run_wget(os.path.join(basedir, section, icondir), urls[icondir])
+            if icondir in urls:
+                _run_wget(os.path.join(basedir, section, icondir), urls[icondir])
 
 
 if __name__ == "__main__":

fdroidserver/rewritemeta.py

@@ -82,9 +82,7 @@ def main():
 
     config = common.read_config(options)
 
-    # Get all apps...
-    allapps = metadata.read_metadata(options.appid)
-    apps = common.read_app_args(options.appid, allapps, False)
+    apps = common.read_app_args(options.appid)
 
     for appid, app in apps.items():
         path = Path(app.metadatapath)

fdroidserver/scanner.py

@@ -829,10 +829,7 @@ def main():
             _get_tool()
         return
 
-    # Read all app and srclib metadata
-
-    allapps = metadata.read_metadata()
-    apps = common.read_app_args(appids, allapps, True)
+    apps = common.read_app_args(appids, allow_vercodes=True)
 
     build_dir = 'build'
     if not os.path.isdir(build_dir):

tests/index.TestCase

@@ -701,8 +701,14 @@ class IndexTest(unittest.TestCase):
         app = apps[appid]
         metadata = index.package_metadata(app, 'repo')
         # files
-        self.assertEqual(36027, metadata['featureGraphic']['en-US']['size'])
-        self.assertEqual(1413, metadata['icon']['en-US']['size'])
+        self.assertEqual(
+            os.path.getsize(f'repo/{appid}/en-US/featureGraphic.png'),
+            metadata['featureGraphic']['en-US']['size'],
+        )
+        self.assertEqual(
+            os.path.getsize(f'repo/{appid}/en-US/icon.png'),
+            metadata['icon']['en-US']['size'],
+        )
         # localized strings
         self.assertEqual({'en-US': 'title'}, metadata['name'])
         self.assertEqual({'en-US': 'video'}, metadata['video'])

tests/lint.TestCase

@@ -438,6 +438,45 @@ class LintTest(unittest.TestCase):
         with self.assertRaises(TypeError):
             fdroidserver.lint.lint_config('mirrors.yml')
 
+    def test_check_certificate_pinned_binaries_empty(self):
+        fdroidserver.common.config = {}
+        app = fdroidserver.metadata.App()
+        app.AllowedAPKSigningKeys = [
+            'a40da80a59d170caa950cf15c18c454d47a39b26989d8b640ecd745ba71bf5dc'
+        ]
+        self.assertEqual(
+            [],
+            list(fdroidserver.lint.check_certificate_pinned_binaries(app)),
+            "when the config is empty, any signing key should be allowed",
+        )
+
+    def test_lint_known_debug_keys_no_match(self):
+        fdroidserver.common.config = {
+            "apk_signing_key_block_list": "a40da80a59d170caa950cf15c18c454d47a39b26989d8b640ecd745ba71bf5dc"
+        }
+        app = fdroidserver.metadata.App()
+        app.AllowedAPKSigningKeys = [
+            '2fd4fd5f54babba4bcb21237809bb653361d0d2583c80964ec89b28a26e9539e'
+        ]
+        self.assertEqual(
+            [],
+            list(fdroidserver.lint.check_certificate_pinned_binaries(app)),
+            "A signing key that does not match one in the config should be allowed",
+        )
+
+    def test_lint_known_debug_keys(self):
+        fdroidserver.common.config = {
+            'apk_signing_key_block_list': 'a40da80a59d170caa950cf15c18c454d47a39b26989d8b640ecd745ba71bf5dc'
+        }
+        app = fdroidserver.metadata.App()
+        app.AllowedAPKSigningKeys = [
+            'a40da80a59d170caa950cf15c18c454d47a39b26989d8b640ecd745ba71bf5dc'
+        ]
+        for warn in fdroidserver.lint.check_certificate_pinned_binaries(app):
+            anywarns = True
+            logging.debug(warn)
+        self.assertTrue(anywarns)
+
 
 class LintAntiFeaturesTest(unittest.TestCase):
     def setUp(self):

tests/nightly.TestCase

@@ -70,7 +70,10 @@ class NightlyTest(unittest.TestCase):
 
     def tearDown(self):
         self.tempdir.cleanup()
-        os.rmdir(self.testroot)
+        try:
+            os.rmdir(self.testroot)
+        except OSError:  # other test modules might have left stuff around
+            pass
 
     def _copy_test_debug_keystore(self):
         self.dot_android.mkdir()

tests/scanner.TestCase

@@ -14,7 +14,6 @@ import unittest
 import uuid
 import yaml
 import zipfile
-import collections
 import pathlib
 from unittest import mock
 from dataclasses import asdict
@@ -787,7 +786,7 @@ class Test_main(unittest.TestCase):
 
             self.exit_func.assert_not_called()
             self.read_app_args_func.assert_called_once_with(
-                ['com.example.app'], collections.OrderedDict(), True
+                ['com.example.app'], allow_vercodes=True
             )
             self.scan_binary_func.assert_not_called()
 

tests/testcommon.py

@@ -18,6 +18,9 @@
 import os
 import sys
 import tempfile
+import unittest
+
+from pathlib import Path
 
 
 class TmpCwd:
@@ -60,3 +63,12 @@ def mkdtemp():
         return tempfile.TemporaryDirectory()
     else:
         return tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+
+
+def mkdir_testfiles(localmodule, test):
+    """Keep the test files in a labeled test dir for easy reference"""
+    testroot = Path(localmodule) / '.testfiles'
+    testroot.mkdir(exist_ok=True)
+    testdir = testroot / unittest.TestCase.id(test)
+    testdir.mkdir(exist_ok=True)
+    return tempfile.mkdtemp(dir=testdir)