🛠️ update.py: finish minimal IPA support

This add a few missing pieces to get IPA support working. (added and
lastUpdated dates + caching for ipa files)

fdroidserver/update.py

@@ -525,7 +525,10 @@ def insert_obbs(repodir, apps, apks):
 
 
 def version_string_to_int(version):
-    """Approximately convert a [Major].[Minor].[Patch] version string
+    """
+    Convert sermver version designation to version code.
+
+    Approximately convert a [Major].[Minor].[Patch] version string
     consisting of numeric characters (0-9) and periods to a number. The
     exponents are chosen such that it still fits in the 64bit JSON/Android range.
     """
@@ -536,46 +539,73 @@ def version_string_to_int(version):
     return major * 10**12 + minor * 10**6 + patch
 
 
-def process_ipa(repodir, apks):
+def parse_ipa(ipa_path, file_size, sha256):
-    """Scan the .ipa files in a given repo directory.
+    from biplist import readPlist
+
+    ipa = {
+        "apkName": os.path.basename(ipa_path),
+        "hash": sha256,
+        "hashType": "sha256",
+        "size": file_size,
+    }
+
+    with zipfile.ZipFile(ipa_path) as ipa_zip:
+        for info in ipa_zip.infolist():
+            if re.match("Payload/[^/]*.app/Info.plist", info.filename):
+                with ipa_zip.open(info) as plist_file:
+                    plist = readPlist(plist_file)
+                    ipa["packageName"] = plist["CFBundleIdentifier"]
+                    # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring
+                    ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"])
+                    ipa["versionName"] = plist["CFBundleShortVersionString"]
+                    ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k}
+    return ipa
+
+
+def scan_repo_for_ipas(apkcache, repodir, knownapks):
+    """Scan for IPA files in a given repo directory.
 
     Parameters
     ----------
+    apkcache
+      cache dictionary containting cached file infos from previous runs
     repodir
       repo directory to scan
-    apps
+    knownapks
-      list of current, valid apps
+      list of all known files, as per metadata.read_metadata
-    apks
+
-      current information on all APKs
+    Returns
+    -------
+    ipas
+      list of file infos for ipa files in ./repo folder
+    cachechanged
+      ture if new ipa files were found and added to `apkcache`
     """
-    def ipaWarnDelete(f, msg):
+    cachechanged = False
-        logging.warning(msg + ' ' + f)
+    ipas = []
-        if options.delete_unknown:
+    for ipa_path in glob.glob(os.path.join(repodir, '*.ipa')):
-            logging.error(_("Deleting unknown file: {path}").format(path=f))
+        ipa_name = os.path.basename(ipa_path)
-            os.remove(f)
 
-    ipas = glob.glob(os.path.join(repodir, '*.ipa'))
+        file_size = os.stat(ipa_path).st_size
-    if ipas:
+        if file_size == 0:
-        from biplist import readPlist
+            raise FDroidException(_('{path} is zero size!')
-        for f in ipas:
+                                  .format(path=ipa_path))
-            ipa = {}
-            apks.append(ipa)
 
-            ipa["apkName"] = os.path.basename(f)
+        sha256 = common.sha256sum(ipa_path)
-            ipa["hash"] = common.sha256sum(f)
+        ipa = apkcache.get(ipa_name, {})
-            ipa["hashType"] = "sha256"
-            ipa["size"] = os.path.getsize(f)
 
-            with zipfile.ZipFile(f) as ipa_zip:
+        if ipa.get('hash') != sha256:
-                for info in ipa_zip.infolist():
+            ipa = parse_ipa(ipa_path, file_size, sha256)
-                    if re.match("Payload/[^/]*.app/Info.plist", info.filename):
+            apkcache[ipa_name] = ipa
-                        with ipa_zip.open(info) as plist_file:
+            cachechanged = True
-                            plist = readPlist(plist_file)
+
-                            ipa["packageName"] = plist["CFBundleIdentifier"]
+        added = knownapks.recordapk(ipa_name, ipa['packageName'])
-                            # https://developer.apple.com/documentation/bundleresources/information_property_list/cfbundleshortversionstring
+        if added:
-                            ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"])
+            ipa['added'] = added
-                            ipa["versionName"] = plist["CFBundleShortVersionString"]
+
-                            ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k}
+        ipas.append(ipa)
+
+    return ipas, cachechanged
 
 
 def translate_per_build_anti_features(apps, apks):
@@ -1175,7 +1205,10 @@ def insert_localized_app_metadata(apps):
 
 
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
-    """Scan a repo for all files with an extension except APK/OBB.
+    """Scan a repo for all files with an extension except APK/OBB/IPA.
+
+    This allows putting all kinds of files into repostories. E.g. Media Files,
+    Zip archives, ...
 
     Parameters
     ----------
@@ -1192,22 +1225,29 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     repo_files = []
     repodir = repodir.encode()
     for name in os.listdir(repodir):
+        # skip files based on file extensions, that are handled elsewhere
         file_extension = common.get_file_extension(name)
         if file_extension in ('apk', 'obb', 'ipa'):
             continue
+
+        # skip source tarballs generated by fdroidserver
         filename = os.path.join(repodir, name)
         name_utf8 = name.decode()
         if filename.endswith(b'_src.tar.gz'):
             logging.debug(_('skipping source tarball: {path}')
                           .format(path=filename.decode()))
             continue
+
+        # skip all other files generated by fdroidserver
         if not common.is_repo_file(filename):
             continue
+
         stat = os.stat(filename)
         if stat.st_size == 0:
             raise FDroidException(_('{path} is zero size!')
                                   .format(path=filename))
 
+        # load file infos from cache if not stale
         shasum = common.sha256sum(filename)
         usecache = False
         if name_utf8 in apkcache:
@@ -1220,6 +1260,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
                               .format(apkfilename=name_utf8))
 
+        # scan file if info wasn't in cache
         if not usecache:
             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
             repo_file = collections.OrderedDict()
@@ -2182,7 +2223,6 @@ def prepare_apps(apps, apks, repodir):
     -------
     the relevant subset of apps (as a deepcopy)
     """
-    process_ipa(repodir, apks)
     apps_with_packages = get_apps_with_packages(apps, apks)
     apply_info_from_latest_apk(apps_with_packages, apks)
     insert_funding_yml_donation_links(apps)
@@ -2309,6 +2349,11 @@ def main():
                                            options.use_date_from_apk)
     cachechanged = cachechanged or fcachechanged
     apks += files
+
+    ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks)
+    cachechanged = cachechanged or icachechanged
+    apks += ipas
+
     appid_has_apks = set()
     appid_has_repo_files = set()
     remove_apks = []