Merge branch 'update-import-fixups' into 'master'

update/import fixups Closes #548 and #472 See merge request fdroid/fdroidserver!584
2024-10-02 09:10:11 +02:00 · 2018-10-12 12:59:43 +00:00 · 2018-10-12 12:59:43 +00:00 · ceb3f6631b
commit ceb3f6631b
parent 9a85f0007c e591a73b82
15 changed files with 224 additions and 25 deletions
--- a/completion/bash-completion
+++ b/completion/bash-completion
@ -186,10 +186,14 @@ __complete_checkupdates() {
 }

 __complete_import() {
-	opts="-u -s -q"
-	lopts="--url --subdir --rev --quiet"
+	opts="-c -h -l -q -s -u -v -W"
+	lopts="--categories --license --quiet --rev --subdir --url"
 	case "${prev}" in
-		-u|--url|-s|--subdir|--rev) return 0;;
+		-c|-l|-s|-u|--categories|--license|--quiet|--rev|--subdir|--url)
+		return 0;;
+		-W)
+		COMPREPLY=( $( compgen -W "error warn ignore" -- $cur ) )
+		return 0;;
 	esac
 	__complete_options
 }
--- a/fdroidserver/import.py
+++ b/fdroidserver/import.py
@ -227,11 +227,11 @@ def main():
    if local_metadata_files != []:
        raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])

+    build = metadata.Build()
    if options.url is None and os.path.isdir('.git'):
        app.AutoName = os.path.basename(os.getcwd())
        app.RepoType = 'git'

-        build = metadata.Build()
        root_dir = get_subdir(os.getcwd())
        if os.path.exists('build.gradle'):
            build.gradle = ['yes']
@ -250,7 +250,6 @@ def main():
        write_local_file = True
    elif options.url:
        root_dir, build_dir = get_metadata_from_url(app, options.url)
-        build = metadata.Build()
        build.commit = '?'
        build.disable = 'Generated by import.py - check/set version fields and commit id'
        write_local_file = False
@ -286,14 +285,14 @@ def main():
        raise FDroidException("Package " + package + " already exists")

    # Create a build line...
-    build.versionName = versionName or '?'
-    build.versionCode = versionCode or '?'
+    build.versionName = versionName or 'Unknown'
+    build.versionCode = versionCode or '0'  # TODO heinous but this is still a str
    if options.subdir:
        build.subdir = options.subdir
    if options.license:
        app.License = options.license
    if options.categories:
-        app.Categories = options.categories
+        app.Categories = options.categories.split(',')
    if os.path.exists(os.path.join(root_dir, 'jni')):
        build.buildjni = ['yes']
    if os.path.exists(os.path.join(root_dir, 'build.gradle')):
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@ -1096,6 +1096,11 @@ def parse_yaml_metadata(mf, app):


 def write_yaml(mf, app):
+    """Write metadata in yaml format.
+
+    :param mf: active file discriptor for writing
+    :param app: app metadata to written to the yaml file
+    """

    # import rumael.yaml and check version
    try:
@ -1176,12 +1181,10 @@ def write_yaml(mf, app):
        return cm

    def _builds_to_yaml(app):
-        fields = ['versionName', 'versionCode']
-        fields.extend(build_flags_order)
        builds = ruamel.yaml.comments.CommentedSeq()
        for build in app.builds:
            b = ruamel.yaml.comments.CommentedMap()
-            for field in fields:
+            for field in build_flags:
                if hasattr(build, field) and getattr(build, field):
                    value = getattr(build, field)
                    if field == 'gradle' and value == ['off']:
--- a/fdroidserver/update.py
+++ b/fdroidserver/update.py
@ -1090,8 +1090,6 @@ def scan_apk(apk_file):
    if 'minSdkVersion' not in apk:
        logging.warning("No SDK version information found in {0}".format(apk_file))
        apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
-    if 'targetSdkVersion' not in apk:
-        apk['targetSdkVersion'] = apk['minSdkVersion']

    # Check for known vulnerabilities
    if has_known_vulnerability(apk_file):
--- a/tests/config.py
+++ b/tests/config.py
@ -17,6 +17,8 @@ archive_description = """
 The repository of older versions of applications from the main demo repository.
 """

+make_current_version_link = False
+
 repo_keyalias = "sova"
 keystore = "keystore.jks"
 keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
--- a/tests/lint.TestCase
+++ b/tests/lint.TestCase
@ -103,6 +103,16 @@ class LintTest(unittest.TestCase):
                    'Multimedia,Graphics',
                ],
            },
+            'WebSite': {
+                'good': [
+                    'https://homepage.com',
+                ],
+                'bad': [
+                    [],
+                    ['nope', ],
+                    29,
+                ],
+            },
        }

        for field, values in fields.items():
--- a/tests/metadata.TestCase
+++ b/tests/metadata.TestCase
@ -165,6 +165,50 @@ class MetadataTest(unittest.TestCase):
            with self.assertRaises(MetaDataException):
                fdroidserver.metadata.parse_yaml_metadata(mf, {})

+    def test_write_yaml_with_placeholder_values(self):
+        mf = io.StringIO()
+
+        app = fdroidserver.metadata.App()
+        app.Categories = ['None']
+        app.SourceCode = "https://gitlab.com/fdroid/fdroidclient.git"
+        app.IssueTracker = "https://gitlab.com/fdroid/fdroidclient/issues"
+        app.RepoType = 'git'
+        app.Repo = 'https://gitlab.com/fdroid/fdroidclient.git'
+        app.AutoUpdateMode = 'None'
+        app.UpdateCheckMode = 'Tags'
+        build = fdroidserver.metadata.Build()
+        build.versionName = 'Unknown'  # taken from fdroidserver/import.py
+        build.versionCode = '0'  # taken from fdroidserver/import.py
+        build.disable = 'Generated by import.py ...'
+        build.commit = 'Unknown'
+        build.gradle = [True]
+        app.builds = [build]
+
+        fdroidserver.metadata.write_yaml(mf, app)
+
+        mf.seek(0)
+        self.assertEqual(mf.read(), textwrap.dedent("""\
+            Categories:
+              - None
+            License: Unknown
+            SourceCode: https://gitlab.com/fdroid/fdroidclient.git
+            IssueTracker: https://gitlab.com/fdroid/fdroidclient/issues
+
+            RepoType: git
+            Repo: https://gitlab.com/fdroid/fdroidclient.git
+
+            Builds:
+              - versionName: Unknown
+                versionCode: 0
+                disable: Generated by import.py ...
+                commit: Unknown
+                gradle:
+                  - true
+
+            AutoUpdateMode: None
+            UpdateCheckMode: Tags
+            """))
+

 if __name__ == "__main__":
    os.chdir(os.path.dirname(__file__))
@ -176,4 +220,4 @@ if __name__ == "__main__":

    newSuite = unittest.TestSuite()
    newSuite.addTest(unittest.makeSuite(MetadataTest))
-    unittest.main(failfast=False)
+    unittest.main(failfast=True)
--- a/tests/metadata/info.zwanenburg.caffeinetile.yml
+++ b/tests/metadata/info.zwanenburg.caffeinetile.yml
@ -0,0 +1,4 @@
+Categories:
+- Development
+Name: Caffeine Tile
+Summary: Test app for extracting icons when an XML one is default
--- a/tests/metadata/no.min.target.sdk.yml
+++ b/tests/metadata/no.min.target.sdk.yml
@ -0,0 +1,3 @@
+Categories: [Development]
+Name: No minSdkVersion or targetSdkVersion
+Summary: An APK without any <uses-sdk> block in AndroidManifest.xml
--- a/tests/repo/categories.txt
+++ b/tests/repo/categories.txt
@ -3,8 +3,8 @@
 Development
 GuardianProject
 Multimedia
-None
 Phone & SMS
 Security
 System
 Time
+tests
--- a/tests/repo/index-v1.json
+++ b/tests/repo/index-v1.json
@ -41,6 +41,19 @@
      "packageName": "souch.smsbypass",
      "lastUpdated": 1524700800000
    },
+    {
+      "categories": [
+        "Development"
+      ],
+      "suggestedVersionCode": "4",
+      "license": "Unknown",
+      "name": "Caffeine Tile",
+      "summary": "Test app for extracting icons when an XML one is default",
+      "added": 1539129600000,
+      "icon": "info.zwanenburg.caffeinetile.4.xml",
+      "packageName": "info.zwanenburg.caffeinetile",
+      "lastUpdated": 1539129600000
+    },
    {
      "categories": [
        "tests"
@ -72,6 +85,19 @@
      "packageName": "fake.ota.update",
      "lastUpdated": 1457568000000
    },
+    {
+      "categories": [
+        "Development"
+      ],
+      "suggestedVersionCode": "987",
+      "license": "Unknown",
+      "name": "No minSdkVersion or targetSdkVersion",
+      "summary": "An APK without any <uses-sdk> block in AndroidManifest.xml",
+      "added": 1539129600000,
+      "icon": "no.min.target.sdk.987.png",
+      "packageName": "no.min.target.sdk",
+      "lastUpdated": 1539129600000
+    },
    {
      "bitcoin": "1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk",
      "categories": [
@ -248,7 +274,6 @@
        "sig": "b4964fd759edaa54e65bb476d0276880",
        "signer": "32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6",
        "size": 18489,
-        "targetSdkVersion": "3",
        "uses-permission": [
          [
            "android.permission.READ_CALENDAR",
@ -290,7 +315,6 @@
        "sig": "b4964fd759edaa54e65bb476d0276880",
        "signer": "32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6",
        "size": 17552,
-        "targetSdkVersion": "3",
        "uses-permission": [
          [
            "android.permission.READ_CALENDAR",
@ -400,6 +424,57 @@
        "versionName": "0.1"
      }
    ],
+    "info.zwanenburg.caffeinetile": [
+      {
+        "added": 1539129600000,
+        "apkName": "info.zwanenburg.caffeinetile_4.apk",
+        "hash": "dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db",
+        "hashType": "sha256",
+        "minSdkVersion": "24",
+        "packageName": "info.zwanenburg.caffeinetile",
+        "sig": "03f9b2f848d22fd1d8d1331e8b1b486d",
+        "signer": "51cfa5c8a743833ad89acf81cb755936876a5c8b8eca54d1ffdcec0cdca25d0e",
+        "size": 11740,
+        "targetSdkVersion": "25",
+        "uses-permission": [
+          [
+            "android.permission.WAKE_LOCK",
+            null
+          ]
+        ],
+        "versionCode": 4,
+        "versionName": "1.3"
+      }
+    ],
+    "no.min.target.sdk": [
+      {
+        "added": 1539129600000,
+        "apkName": "no.min.target.sdk_987.apk",
+        "hash": "e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c",
+        "hashType": "sha256",
+        "minSdkVersion": 3,
+        "packageName": "no.min.target.sdk",
+        "sig": "b4964fd759edaa54e65bb476d0276880",
+        "signer": "32a23624c201b949f085996ba5ed53d40f703aca4989476949cae891022e0ed6",
+        "size": 14102,
+        "uses-permission": [
+          [
+            "android.permission.WRITE_EXTERNAL_STORAGE",
+            null
+          ],
+          [
+            "android.permission.READ_PHONE_STATE",
+            null
+          ],
+          [
+            "android.permission.READ_EXTERNAL_STORAGE",
+            null
+          ]
+        ],
+        "versionCode": 987,
+        "versionName": "1.2-fake"
+      }
+    ],
    "obb.main.oldversion": [
      {
        "added": 1388448000000,
@ -602,4 +677,4 @@
      }
    ]
  }
-}
+}
--- a/tests/repo/index.xml
+++ b/tests/repo/index.xml
@ -39,6 +39,35 @@
 			<permissions>READ_CONTACTS,READ_EXTERNAL_STORAGE,RECEIVE_SMS,SEND_SMS,VIBRATE,WRITE_EXTERNAL_STORAGE</permissions>
 		</package>
 	</application>
+	<application id="info.zwanenburg.caffeinetile">
+		<id>info.zwanenburg.caffeinetile</id>
+		<added>2018-10-10</added>
+		<lastupdated>2018-10-10</lastupdated>
+		<name>Caffeine Tile</name>
+		<summary>Test app for extracting icons when an XML one is default</summary>
+		<icon>info.zwanenburg.caffeinetile.4.xml</icon>
+		<desc>&lt;p&gt;No description available&lt;/p&gt;</desc>
+		<license>Unknown</license>
+		<categories>Development</categories>
+		<category>Development</category>
+		<web></web>
+		<source></source>
+		<tracker></tracker>
+		<marketversion></marketversion>
+		<marketvercode>4</marketvercode>
+		<package>
+			<version>1.3</version>
+			<versioncode>4</versioncode>
+			<apkname>info.zwanenburg.caffeinetile_4.apk</apkname>
+			<hash type="sha256">dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db</hash>
+			<size>11740</size>
+			<sdkver>24</sdkver>
+			<targetSdkVersion>25</targetSdkVersion>
+			<added>2018-10-10</added>
+			<sig>03f9b2f848d22fd1d8d1331e8b1b486d</sig>
+			<permissions>WAKE_LOCK</permissions>
+		</package>
+	</application>
 	<application id="duplicate.permisssions">
 		<id>duplicate.permisssions</id>
 		<added>2017-12-22</added>
@ -95,6 +124,34 @@
 			<added>2016-03-10</added>
 		</package>
 	</application>
+	<application id="no.min.target.sdk">
+		<id>no.min.target.sdk</id>
+		<added>2018-10-10</added>
+		<lastupdated>2018-10-10</lastupdated>
+		<name>No minSdkVersion or targetSdkVersion</name>
+		<summary>An APK without any &lt;uses-sdk&gt; block in AndroidManifest.xml</summary>
+		<icon>no.min.target.sdk.987.png</icon>
+		<desc>&lt;p&gt;No description available&lt;/p&gt;</desc>
+		<license>Unknown</license>
+		<categories>Development</categories>
+		<category>Development</category>
+		<web></web>
+		<source></source>
+		<tracker></tracker>
+		<marketversion></marketversion>
+		<marketvercode>987</marketvercode>
+		<package>
+			<version>1.2-fake</version>
+			<versioncode>987</versioncode>
+			<apkname>no.min.target.sdk_987.apk</apkname>
+			<hash type="sha256">e2e1dc1d550df2b5bc383860139207258645b5540abeccd305ed8b2cb6459d2c</hash>
+			<size>14102</size>
+			<sdkver>3</sdkver>
+			<added>2018-10-10</added>
+			<sig>b4964fd759edaa54e65bb476d0276880</sig>
+			<permissions>READ_EXTERNAL_STORAGE,READ_PHONE_STATE,WRITE_EXTERNAL_STORAGE</permissions>
+		</package>
+	</application>
 	<application id="obb.main.oldversion">
 		<id>obb.main.oldversion</id>
 		<added>2013-12-31</added>
@ -269,7 +326,6 @@
 			<hash type="sha256">c809bdff83715fbf919f3840ee09869b038e209378b906e135ee40d3f0e1f075</hash>
 			<size>18489</size>
 			<sdkver>3</sdkver>
-			<targetSdkVersion>3</targetSdkVersion>
 			<added>2017-06-23</added>
 			<sig>b4964fd759edaa54e65bb476d0276880</sig>
 			<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>
@ -281,7 +337,6 @@
 			<hash type="sha256">665d03d61ebc642289fda697f71a59305b0202b16cafc5ffdae91cbe91f0b25d</hash>
 			<size>17552</size>
 			<sdkver>3</sdkver>
-			<targetSdkVersion>3</targetSdkVersion>
 			<added>2017-06-23</added>
 			<sig>b4964fd759edaa54e65bb476d0276880</sig>
 			<permissions>READ_CALENDAR,READ_EXTERNAL_STORAGE,READ_PHONE_STATE,RECEIVE_BOOT_COMPLETED,WRITE_EXTERNAL_STORAGE</permissions>
--- a/tests/repo/no.min.target.sdk_987.apk
+++ b/tests/repo/no.min.target.sdk_987.apk
--- a/tests/stats/known_apks.txt
+++ b/tests/stats/known_apks.txt
@ -4,6 +4,8 @@ com.politedroid_5.apk com.politedroid 2017-06-23
 com.politedroid_6.apk com.politedroid 2017-06-23
 duplicate.permisssions_9999999.apk duplicate.permisssions 2017-12-22
 fake.ota.update_1234.zip fake.ota.update 2016-03-10
+info.zwanenburg.caffeinetile_4.apk info.zwanenburg.caffeinetile 2018-10-10
+no.min.target.sdk_987.apk no.min.target.sdk 2018-10-10
 obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
 obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12
 obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01
--- a/tests/update.TestCase
+++ b/tests/update.TestCase
@ -254,14 +254,14 @@ class UpdateTest(unittest.TestCase):
        apps = fdroidserver.metadata.read_metadata(xref=True)
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
-        self.assertEqual(len(apks), 14)
+        self.assertEqual(len(apks), 15)
        apk = apks[0]
        self.assertEqual(apk['packageName'], 'com.politedroid')
        self.assertEqual(apk['versionCode'], 3)
        self.assertEqual(apk['minSdkVersion'], '3')
-        self.assertEqual(apk['targetSdkVersion'], '3')
+        self.assertIsNone(apk.get('targetSdkVersion'))
        self.assertFalse('maxSdkVersion' in apk)
-        apk = apks[6]
+        apk = apks[7]
        self.assertEqual(apk['packageName'], 'obb.main.oldversion')
        self.assertEqual(apk['versionCode'], 1444412523)
        self.assertEqual(apk['minSdkVersion'], '4')
@ -321,7 +321,7 @@ class UpdateTest(unittest.TestCase):

        fdroidserver.update.options.clean = False
        read_from_json = fdroidserver.update.get_cache()
-        self.assertEqual(16, len(read_from_json))
+        self.assertEqual(17, len(read_from_json))
        for f in glob.glob('repo/*.apk'):
            self.assertTrue(os.path.basename(f) in read_from_json)

@ -623,7 +623,7 @@ class UpdateTest(unittest.TestCase):
        knownapks = fdroidserver.common.KnownApks()
        apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
        fdroidserver.update.translate_per_build_anti_features(apps, apks)
-        self.assertEqual(len(apks), 14)
+        self.assertEqual(len(apks), 15)
        foundtest = False
        for apk in apks:
            if apk['packageName'] == 'com.politedroid' and apk['versionCode'] == 3: