Merge branch 'more-api-fixes' into 'master'

do not assume `app` is an App instance, support API usage

See merge request fdroid/fdroidserver!826

.gitlab-ci.yml

@@ -247,9 +247,13 @@ gradle:
 # this tests the basic setup of the 'fdroid build' CI job in fdroiddata
 fdroiddata fdroid build:
   image: registry.gitlab.com/fdroid/ci-images-client:latest
-  only:
+  rules:
-    changes:
+    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
-      - buildserver/provision-apt-get-install
+      changes:
+        - buildserver/provision-apt-get-install
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      changes:
+        - buildserver/provision-apt-get-install
   script:
     - bash buildserver/provision-apt-get-install http://deb.debian.org/debian
     - apt-get dist-upgrade

fdroidserver/update.py

@@ -1891,7 +1891,7 @@ def apply_info_from_latest_apk(apps, apks):
                     bestver = apk['versionCode']
                     bestapk = apk
 
-                if app['NoSourceSince']:
+                if app.get('NoSourceSince'):
                     apk['antiFeatures'].add('NoSourceSince')
 
         if not app['added']:
@@ -1901,12 +1901,12 @@ def apply_info_from_latest_apk(apps, apks):
 
         if bestver == UNSET_VERSION_CODE:
 
-            if app['Name'] is None:
+            if app.get('Name') is None:
                 app['Name'] = app['AutoName'] or appid
             app['icon'] = None
             logging.debug("Application " + appid + " has no packages")
         else:
-            if app['Name'] is None:
+            if app.get('Name') is None:
                 app['Name'] = bestapk['name']
             app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
             if app['CurrentVersionCode'] is None:
@@ -2095,10 +2095,10 @@ def read_added_date_from_all_apks(apps, apks):
         for apk in apks:
             if apk['packageName'] == appid:
                 if 'added' in apk:
-                    if not app.added or apk['added'] < app.added:
+                    if not app.get('added') or apk['added'] < app['added']:
-                        app.added = apk['added']
+                        app['added'] = apk['added']
-                    if not app.lastUpdated or apk['added'] > app.lastUpdated:
+                    if not app.get('lastUpdated') or apk['added'] > app['lastUpdated']:
-                        app.lastUpdated = apk['added']
+                        app['lastUpdated'] = apk['added']
 
 
 def read_names_from_apks(apps, apks):

tests/update.TestCase

@@ -18,6 +18,7 @@ import yaml
 import zipfile
 import textwrap
 from binascii import unhexlify
+from datetime import datetime
 from distutils.version import LooseVersion
 from testcommon import TmpCwd
 
@@ -57,6 +58,12 @@ DONATION_FIELDS = (
 )
 
 
+class Options:
+    allow_disabled_algorithms = False
+    clean = False
+    rename_apks = False
+
+
 class UpdateTest(unittest.TestCase):
     '''fdroid update'''
 
@@ -452,6 +459,56 @@ class UpdateTest(unittest.TestCase):
         reset = fdroidserver.update.get_cache()
         self.assertEqual(2, len(reset))
 
+    def test_scan_repo_files(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+        os.mkdir('repo')
+        os.mkdir('stats')
+        with open(os.path.join('stats', 'known_apks.txt'), 'w') as fp:
+            fp.write('se.manyver_30.apk se.manyver 2018-10-10\n')
+        filename = 'Virgin-islands-british_centralamerica_2.obf.zip'
+        shutil.copy(os.path.join(self.basedir, filename), 'repo')
+        knownapks = fdroidserver.common.KnownApks()
+        files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False)
+        knownapks.writeifchanged()
+        self.assertTrue(fcachechanged)
+
+        info = files[0]
+        self.assertEqual(filename, info['apkName'])
+        self.assertEqual(datetime, type(info['added']))
+        self.assertEqual(os.path.getsize(os.path.join('repo', filename)), info['size'])
+        self.assertEqual('402ee0799d5da535276b5a3672fb049d6df3e1727cfb35369c8962c4a42cac3d',
+                         info['packageName'])
+
+    def test_read_added_date_from_all_apks(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+        fdroidserver.update.options = Options
+        os.chdir(os.path.join(localmodule, 'tests'))
+        apps = fdroidserver.metadata.read_metadata()
+        knownapks = fdroidserver.common.KnownApks()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
+        fdroidserver.update.read_added_date_from_all_apks(apps, apks)
+
+    def test_apply_info_from_latest_apk(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+        fdroidserver.update.options = Options
+        os.chdir(os.path.join(localmodule, 'tests'))
+        apps = fdroidserver.metadata.read_metadata()
+        knownapks = fdroidserver.common.KnownApks()
+        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks)
+        fdroidserver.update.apply_info_from_latest_apk(apps, apks)
+
     def test_scan_apk(self):
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
@@ -599,6 +656,7 @@ class UpdateTest(unittest.TestCase):
             _, apk, cachechanged = fdroidserver.update.process_apk({}, apkName, 'repo', knownapks,
                                                                    False)
             # Don't care about the date added to the repo and relative apkName
+            self.assertEqual(datetime, type(apk['added']))
             del apk['added']
             del apk['apkName']