Merge branch 'alt-store' into 'master'

iOS metadata + alt-store source support

See merge request fdroid/fdroidserver!1465

fdroidserver/common.py

@@ -4165,7 +4165,12 @@ def is_repo_file(filename, for_gpg_signing=False):
         b'index_unsigned.jar',
     ]
     if not for_gpg_signing:
-        ignore_files += [b'entry.json', b'index-v1.json', b'index-v2.json']
+        ignore_files += [
+            b'altstore-index.json',
+            b'entry.json',
+            b'index-v1.json',
+            b'index-v2.json',
+        ]
 
     return (
         os.path.isfile(filename)

fdroidserver/deploy.py

@@ -71,6 +71,7 @@ def _get_index_excludes(repo_section):
 
     """
     indexes = [
+        os.path.join(repo_section, 'altstore-index.json'),
         os.path.join(repo_section, 'entry.jar'),
         os.path.join(repo_section, 'entry.json'),
         os.path.join(repo_section, 'entry.json.asc'),

fdroidserver/index.py

@@ -125,6 +125,13 @@ def make(apps, apks, repodir, archive):
     make_v2(sortedapps, apks, repodir, repodict, requestsdict,
             fdroid_signing_key_fingerprints, archive)
     make_website(sortedapps, repodir, repodict)
+    make_altstore(
+        sortedapps,
+        apks,
+        common.config,
+        repodir,
+        pretty=common.options.pretty,
+    )
 
 
 def _should_file_be_generated(path, magic_string):
@@ -1750,3 +1757,96 @@ def get_public_key_from_jar(jar):
     public_key_fingerprint = common.get_cert_fingerprint(public_key).replace(' ', '')
 
     return public_key, public_key_fingerprint
+
+
+def make_altstore(apps, apks, config, repodir, pretty=False):
+    """Assemble altstore-index.json for iOS (.ipa) apps.
+
+    builds index files based on:
+    https://faq.altstore.io/distribute-your-apps/make-a-source
+    https://faq.altstore.io/distribute-your-apps/updating-apps
+    """
+    indent = 2 if pretty else None
+    # for now alt-store support is english only
+    for lang in ['en']:
+
+        # prepare minimal altstore index
+        idx = {
+            'name': config['repo_name'],
+            "apps": [],
+            "news": [],
+        }
+
+        # add optional values if available
+        # idx["subtitle"] F-Droid doesn't have a corresponding value
+        if config.get("repo_description"):
+            idx['description'] = config['repo_description']
+        if (Path(repodir) / 'icons' / config['repo_icon']).exists():
+            idx['iconURL'] = f"{config['repo_url']}/icons/{config['repo_icon']}"
+        # idx["headerURL"] F-Droid doesn't have a corresponding value
+        # idx["website"] F-Droid doesn't have a corresponding value
+        # idx["patreonURL"] F-Droid doesn't have a corresponding value
+        # idx["tintColor"] F-Droid doesn't have a corresponding value
+        # idx["featuredApps"] = [] maybe mappable to F-Droids what's new?
+
+        # assemble "apps"
+        for packageName, app in apps.items():
+            app_name = app.get("Name") or app.get("AutoName")
+            icon_url = "{}{}".format(
+                config['repo_url'],
+                app.get('iconv2', {}).get(DEFAULT_LOCALE, {}).get('name', ''),
+            )
+            screenshot_urls = [
+                "{}{}".format(config["repo_url"], s["name"])
+                for s in app.get("screenshots", {})
+                .get("phone", {})
+                .get(DEFAULT_LOCALE, {})
+            ]
+
+            a = {
+                "name": app_name,
+                'bundleIdentifier': packageName,
+                'developerName': app.get("AuthorName") or f"{app_name} team",
+                'iconURL': icon_url,
+                "localizedDescription": "",
+                'appPermissions': {
+                    "entitlements": set(),
+                    "privacy": {},
+                },
+                'versions': [],
+            }
+
+            if app.get('summary'):
+                a['subtitle'] = app['summary']
+            # a["tintColor"] F-Droid doesn't have a corresponding value
+            # a["category"] F-Droid doesn't have a corresponding value
+            # a['patreon'] F-Droid doesn't have a corresponding value
+            a["screenshots"] = screenshot_urls
+
+            # populate 'versions'
+            for apk in apks:
+                last4 = apk.get('apkName', '').lower()[-4:]
+                if apk['packageName'] == packageName and last4 == '.ipa':
+                    v = {
+                        "version": apk["versionName"],
+                        "date": apk["added"].isoformat(),
+                        "downloadURL": f"{config['repo_url']}/{apk['apkName']}",
+                        "size": apk['size'],
+                    }
+
+                    # v['localizedDescription'] maybe what's new text?
+                    v["minOSVersion"] = apk["ipa_MinimumOSVersion"]
+                    v["maxOSVersion"] = apk["ipa_DTPlatformVersion"]
+
+                    # writing this spot here has the effect that always the
+                    # permissions of the latest processed permissions list used
+                    a['appPermissions']['privacy'] = apk['ipa_permissions']
+                    a['appPermissions']['entitlements'] = list(apk['ipa_entitlements'])
+
+                    a['versions'].append(v)
+
+            if len(a['versions']) > 0:
+                idx['apps'].append(a)
+
+        with open(Path(repodir) / 'altstore-index.json', "w", encoding="utf-8") as f:
+            json.dump(idx, f, indent=indent)

fdroidserver/update.py

@@ -34,6 +34,7 @@ import json
 import time
 import yaml
 import copy
+import asn1crypto.cms
 import defusedxml.ElementTree as ElementTree
 from datetime import datetime, timezone
 from argparse import ArgumentParser
@@ -544,25 +545,105 @@ def version_string_to_int(version):
     return major * 10**12 + minor * 10**6 + patch
 
 
+# iOS app permissions, source:
+# https://developer.apple.com/documentation/bundleresources/information_property_list/protected_resources
+IPA_PERMISSIONS = [
+    "NSBluetoothAlwaysUsageDescription",
+    "NSBluetoothPeripheralUsageDescription",
+    "NSCalendarsFullAccessUsageDescription",
+    "NSCalendarsWriteOnlyAccessUsageDescription",
+    "NSRemindersFullAccessUsageDescription",
+    "NSCameraUsageDescription",
+    "NSMicrophoneUsageDescription",
+    "NSContactsUsageDescription",
+    "NSFaceIDUsageDescription",
+    "NSDesktopFolderUsageDescription",
+    "NSDocumentsFolderUsageDescription",
+    "NSDownloadsFolderUsageDescription",
+    "NSNetworkVolumesUsageDescription",
+    "NSNetworkVolumesUsageDescription",
+    "NSRemovableVolumesUsageDescription",
+    "NSRemovableVolumesUsageDescription",
+    "NSFileProviderDomainUsageDescription",
+    "NSGKFriendListUsageDescription",
+    "NSHealthClinicalHealthRecordsShareUsageDescription",
+    "NSHealthShareUsageDescription",
+    "NSHealthUpdateUsageDescription",
+    "NSHomeKitUsageDescription",
+    "NSLocationAlwaysAndWhenInUseUsageDescription",
+    "NSLocationUsageDescription",
+    "NSLocationWhenInUseUsageDescription",
+    "NSLocationAlwaysUsageDescription",
+    "NSAppleMusicUsageDescription",
+    "NSMotionUsageDescription",
+    "NSFallDetectionUsageDescription",
+    "NSLocalNetworkUsageDescription",
+    "NSNearbyInteractionUsageDescription",
+    "NSNearbyInteractionAllowOnceUsageDescription",
+    "NFCReaderUsageDescription",
+    "NSPhotoLibraryAddUsageDescription",
+    "NSPhotoLibraryUsageDescription",
+    "NSAppDataUsageDescription",
+    "NSUserTrackingUsageDescription",
+    "NSAppleEventsUsageDescription",
+    "NSSystemAdministrationUsageDescription",
+    "NSSensorKitUsageDescription",
+    "NSSiriUsageDescription",
+    "NSSpeechRecognitionUsageDescription",
+    "NSVideoSubscriberAccountUsageDescription",
+    "NSWorldSensingUsageDescription",
+    "NSHandsTrackingUsageDescription",
+    "NSIdentityUsageDescription",
+    "NSCalendarsUsageDescription",
+    "NSRemindersUsageDescription",
+]
+
+
 def parse_ipa(ipa_path, file_size, sha256):
-    from biplist import readPlist
+    import biplist
 
     ipa = {
         "apkName": os.path.basename(ipa_path),
         "hash": sha256,
         "hashType": "sha256",
         "size": file_size,
+        "ipa_entitlements": set(),
+        "ipa_permissions": {},
     }
 
     with zipfile.ZipFile(ipa_path) as ipa_zip:
         for info in ipa_zip.infolist():
             if re.match("Payload/[^/]*.app/Info.plist", info.filename):
                 with ipa_zip.open(info) as plist_file:
-                    plist = readPlist(plist_file)
+                    plist = biplist.readPlist(plist_file)
+                    ipa["name"] = plist['CFBundleName']
                     ipa["packageName"] = plist["CFBundleIdentifier"]
                     # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring
                     ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"])
                     ipa["versionName"] = plist["CFBundleShortVersionString"]
+                    ipa["ipa_MinimumOSVersion"] = plist['MinimumOSVersion']
+                    ipa["ipa_DTPlatformVersion"] = plist['DTPlatformVersion']
+                    for ipap in IPA_PERMISSIONS:
+                        if ipap in plist:
+                            ipa["ipa_permissions"][ipap] = str(plist[ipap])
+            if info.filename.endswith("/embedded.mobileprovision"):
+                print("parsing", info.filename)
+                with ipa_zip.open(info) as mopro_file:
+                    mopro_content_info = asn1crypto.cms.ContentInfo.load(
+                        mopro_file.read()
+                    )
+                    mopro_payload_info = mopro_content_info['content']
+                    mopro_payload = mopro_payload_info['encap_content_info'][
+                        'content'
+                    ].native
+                    mopro = biplist.readPlistFromString(mopro_payload)
+                    # https://faq.altstore.io/distribute-your-apps/make-a-source#entitlements-array-of-strings
+                    for entitlement in mopro.get('Entitlements', {}).keys():
+                        if entitlement not in [
+                            "com.app.developer.team-identifier",
+                            'application-identifier'
+                        ]:
+                            ipa["ipa_entitlements"].add(entitlement)
     return ipa
 
 
@@ -592,8 +673,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks):
 
         file_size = os.stat(ipa_path).st_size
         if file_size == 0:
-            raise FDroidException(_('{path} is zero size!')
+            raise FDroidException(_('{path} is zero size!').format(path=ipa_path))
-                                  .format(path=ipa_path))
 
         sha256 = common.sha256sum(ipa_path)
         ipa = apkcache.get(ipa_name, {})
@@ -1346,11 +1426,28 @@ def insert_localized_ios_app_metadata(apps_with_packages):
                     for metadata_file in (lang_dir).iterdir():
                         key = FASTLANE_IOS_MAP.get(metadata_file.name)
                         if key:
-                            fdroidserver.update._set_localized_text_entry(app, locale, key, metadata_file)
+                            fdroidserver.update._set_localized_text_entry(
+                                app, locale, key, metadata_file
+                            )
 
         screenshots = fdroidserver.update.discover_ios_screenshots(fastlane_dir)
         fdroidserver.update.copy_ios_screenshots_to_repo(screenshots, package_name)
 
+        # lookup icons, copy them and put them into app
+        icon_path = _get_ipa_icon(Path('build') / package_name)
+        icon_dest = Path('repo') / package_name / 'icon.png'  # for now just assume png
+        icon_stat = os.stat(icon_path)
+        app['iconv2'] = {
+            DEFAULT_LOCALE: {
+                'name': str(icon_dest).lstrip('repo'),
+                'sha256': common.sha256sum(icon_dest),
+                'size': icon_stat.st_size,
+            }
+        }
+        if not icon_dest.exists():
+            icon_dest.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(icon_path, icon_dest)
+
 
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     """Scan a repo for all files with an extension except APK/OBB/IPA.
@@ -1548,6 +1645,60 @@ def _get_apk_icons_src(apkfile, icon_name):
     return icons_src
 
 
+def _get_ipa_icon(src_dir):
+    """Search source directory of an IPA project for the app icon."""
+    # parse app icon name from project config file
+    src_dir = Path(src_dir)
+    prj = next(src_dir.glob("**/project.pbxproj"), None)
+    if not prj or not prj.exists():
+        return
+
+    icon_name = _parse_from_pbxproj(prj, 'ASSETCATALOG_COMPILER_APPICON_NAME')
+    if not icon_name:
+        return
+
+    icon_dir = next(src_dir.glob(f'**/{icon_name}.appiconset'), None)
+    if not icon_dir:
+        return
+
+    with open(icon_dir / "Contents.json") as f:
+        cntnt = json.load(f)
+
+    fname = None
+    fsize = 0
+    for image in cntnt['images']:
+        s = float(image.get("size", "0x0").split("x")[0])
+        if image.get('scale') == "1x" and s > fsize and s <= 128:
+            fname = image['filename']
+            fsize = s
+
+    return str(icon_dir / fname)
+
+
+def _parse_from_pbxproj(pbxproj_path, key):
+    """Parse values from apple project files.
+
+    This is a naive regex based parser. Should this proofe to unreliable we
+    might want to consider using a dedicated pbxproj parser:
+    https://pypi.org/project/pbxproj/
+
+    e.g. when looking for key 'ASSETCATALOG_COMPILER_APPICON_NAME'
+    This function will extract 'MyIcon' from if the provided file
+    contains this line:
+
+        ASSETCATALOG_COMPILER_APPICON_NAME = MyIcon;
+
+    returns None if parsing for that value didn't yield anything
+    """
+    r = re.compile(f"\\s*{key}\\s*=\\s*(?P<value>[a-zA-Z0-9-_]+)\\s*;\\s*")
+    with open(pbxproj_path, 'r', encoding='utf-8') as f:
+        for line in f.readlines():
+            m = r.match(line)
+            if m:
+                return m.group("value")
+    return None
+
+
 def _sanitize_sdk_version(value):
     """Sanitize the raw values from androguard to handle bad values.
 

tests/deploy.TestCase

@@ -112,6 +112,8 @@ class DeployTest(unittest.TestCase):
             fdroidserver.deploy.update_serverwebroot('serverwebroot', 'repo')
 
     def test_update_serverwebroot_make_cur_version_link(self):
+        self.maxDiff = None
+
         # setup parameters for this test run
         fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_checksum = True
@@ -137,6 +139,8 @@ class DeployTest(unittest.TestCase):
                         '--safe-links',
                         '--quiet',
                         '--exclude',
+                        'repo/altstore-index.json',
+                        '--exclude',
                         'repo/entry.jar',
                         '--exclude',
                         'repo/entry.json',
@@ -232,6 +236,8 @@ class DeployTest(unittest.TestCase):
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
                         '--exclude',
+                        'archive/altstore-index.json',
+                        '--exclude',
                         'archive/entry.jar',
                         '--exclude',
                         'archive/entry.json',

tests/index.TestCase

@@ -829,6 +829,96 @@ class IndexTest(unittest.TestCase):
             index.add_mirrors_to_repodict('repo', repodict)
 
 
+class AltstoreIndexTest(unittest.TestCase):
+    def test_make_altstore(self):
+        self.maxDiff = None
+
+        apps = {
+            "app.fake": {
+                "AutoName": "Fake App",
+                "AuthorName": "Fake Author",
+                "iconv2": {"en_US": "fake_icon.png"},
+            }
+        }
+        apks = [
+            {
+                "packageName": "app.fake",
+                "apkName": "app.fake_123.ipa",
+                "versionName": "v123",
+                "added": datetime.datetime(2000, 2, 2, 2, 2, 2),
+                "size": 123,
+                "ipa_MinimumOSVersion": "10.0",
+                "ipa_DTPlatformVersion": "12.0",
+                "ipa_permissions": [
+                    "NSCameraUsageDescription",
+                    "NSDocumentsFolderUsageDescription",
+                ],
+                "ipa_entitlements": [
+                    "com.apple.developer.team-identifier",
+                    "com.apple.developer.web-browser",
+                    "keychain-access-groups",
+                ],
+            },
+        ]
+        config = {
+            "repo_icon": "fake_repo_icon.png",
+            "repo_name": "fake_repo",
+            "repo_url": "gopher://fake-repo.com/fdroid/repo",
+        }
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            repodir = Path(tmpdir) / 'repo'
+            repodir.mkdir()
+
+            fdroidserver.index.make_altstore(
+                apps,
+                apks,
+                config,
+                repodir,
+                True,
+            )
+
+            with open(repodir / "altstore-index.json", 'r') as f:
+                self.assertDictEqual(
+                    {
+                        "apps": [
+                            {
+                                "appPermissions": {
+                                    "entitlements": [
+                                        'com.apple.developer.team-identifier',
+                                        'com.apple.developer.web-browser',
+                                        'keychain-access-groups',
+                                    ],
+                                    'privacy': [
+                                        'NSCameraUsageDescription',
+                                        'NSDocumentsFolderUsageDescription',
+                                    ],
+                                },
+                                'bundleIdentifier': 'app.fake',
+                                'developerName': 'Fake Author',
+                                'iconURL': 'gopher://fake-repo.com/fdroid/repo',
+                                'localizedDescription': '',
+                                'name': 'Fake App',
+                                'screenshots': [],
+                                'versions': [
+                                    {
+                                        'date': '2000-02-02T02:02:02',
+                                        'downloadURL': 'gopher://fake-repo.com/fdroid/repo/app.fake_123.ipa',
+                                        'maxOSVersion': '12.0',
+                                        'minOSVersion': '10.0',
+                                        'size': 123,
+                                        'version': 'v123',
+                                    }
+                                ],
+                            },
+                        ],
+                        'name': 'fake_repo',
+                        'news': [],
+                    },
+                    json.load(f),
+                )
+
+
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
 
@@ -845,4 +935,5 @@ if __name__ == "__main__":
 
     newSuite = unittest.TestSuite()
     newSuite.addTest(unittest.makeSuite(IndexTest))
+    newSuite.addTest(unittest.makeSuite(AltstoreIndexTest))
     unittest.main(failfast=False)

tests/update.TestCase

@@ -1936,7 +1936,10 @@ class UpdateTest(unittest.TestCase):
             index['repo'][CATEGORIES_CONFIG_NAME],
         )
 
+
+class TestParseIpa(unittest.TestCase):
     def test_parse_ipa(self):
+        self.maxDiff = None
         try:
             import biplist  # Fedora does not have a biplist package
 
@@ -1959,6 +1962,27 @@ class UpdateTest(unittest.TestCase):
                 'size': 'fake_size',
                 'versionCode': 1000000000001,
                 'versionName': '1.0.1',
+                'ipa_DTPlatformVersion': '16.4',
+                'ipa_MinimumOSVersion': '15.0',
+                'ipa_entitlements': set(),
+                'ipa_permissions': {
+                    'NSCameraUsageDescription':
+                        'Please allow access to your '
+                        'camera, if you want to '
+                        'create photos or videos for '
+                        'direct sharing.',
+                    'NSMicrophoneUsageDescription':
+                        'Please allow access to '
+                        'your microphone, if you '
+                        'want to create videos '
+                        'for direct sharing.',
+                    'NSPhotoLibraryUsageDescription':
+                        'Please allow access to '
+                        'your photo library, if '
+                        'you want to share '
+                        'photos.',
+                },
+                'name': 'OnionShare',
             },
         )
 
@@ -2201,6 +2225,51 @@ class TestCopyIosScreenshotsToRepo(unittest.TestCase):
         )
 
 
+class TestGetIpaIcon(unittest.TestCase):
+    def test_get_ipa_icon(self):
+        self.maxDiff = None
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmpdir = Path(tmpdir)
+            (tmpdir / 'OnionBrowser.xcodeproj').mkdir()
+            with open(tmpdir / 'OnionBrowser.xcodeproj/project.pbxproj', "w") as f:
+                f.write("")
+            icondir = tmpdir / "fake_icon.appiconset"
+            icondir.mkdir()
+            with open(icondir / "Contents.json", "w", encoding="utf-8") as f:
+                f.write("""
+                    {"images": [
+                        {"scale": "2x", "size": "128x128", "filename": "nope"},
+                        {"scale": "1x", "size": "512x512", "filename": "nope"},
+                        {"scale": "1x", "size": "16x16", "filename": "nope"},
+                        {"scale": "1x", "size": "32x32", "filename": "yep"}
+                    ]}
+                """)
+
+            pfp = mock.Mock(return_value="fake_icon")
+            with mock.patch("fdroidserver.update._parse_from_pbxproj", pfp):
+                p = fdroidserver.update._get_ipa_icon(tmpdir)
+                self.assertEqual(str(icondir / "yep"), p)
+
+
+class TestParseFromPbxproj(unittest.TestCase):
+    def test_parse_from_pbxproj(self):
+        self.maxDiff = None
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with open(Path(tmpdir) / "asdf.pbxproj", 'w', encoding="utf-8") as f:
+                f.write("""
+                    230jfaod=flc'
+                    ASSETCATALOG_COMPILER_APPICON_NAME = MyIcon;
+                    cm opa1c p[m
+                """)
+            v = fdroidserver.update._parse_from_pbxproj(
+                Path(tmpdir) / "asdf.pbxproj",
+                "ASSETCATALOG_COMPILER_APPICON_NAME"
+            )
+            self.assertEqual(v, "MyIcon")
+
+
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))
 
@@ -2221,4 +2290,5 @@ if __name__ == "__main__":
     newSuite.addTest(unittest.makeSuite(TestParseIosScreenShotName))
     newSuite.addTest(unittest.makeSuite(TestInsertLocalizedIosAppMetadata))
     newSuite.addTest(unittest.makeSuite(TestDiscoverIosScreenshots))
+    newSuite.addTest(unittest.makeSuite(TestGetIpaIcon))
     unittest.main(failfast=False)