🛠️ update.py: finish minimal IPA support

This add a few missing pieces to get IPA support working. (added and
lastUpdated dates + caching for ipa files)

fdroidserver/update.py

@@ -525,7 +525,10 @@ def insert_obbs(repodir, apps, apks):
 
 
 def version_string_to_int(version):
-    """Approximately convert a [Major].[Minor].[Patch] version string
+    """
+    Convert sermver version designation to version code.
+
+    Approximately convert a [Major].[Minor].[Patch] version string
     consisting of numeric characters (0-9) and periods to a number. The
     exponents are chosen such that it still fits in the 64bit JSON/Android range.
     """
@@ -536,37 +539,17 @@ def version_string_to_int(version):
     return major * 10**12 + minor * 10**6 + patch
 
 
-def process_ipa(repodir, apks):
+def parse_ipa(ipa_path, file_size, sha256):
-    """Scan the .ipa files in a given repo directory.
-
-    Parameters
-    ----------
-    repodir
-      repo directory to scan
-    apps
-      list of current, valid apps
-    apks
-      current information on all APKs
-    """
-    def ipaWarnDelete(f, msg):
-        logging.warning(msg + ' ' + f)
-        if options.delete_unknown:
-            logging.error(_("Deleting unknown file: {path}").format(path=f))
-            os.remove(f)
-
-    ipas = glob.glob(os.path.join(repodir, '*.ipa'))
-    if ipas:
     from biplist import readPlist
-        for f in ipas:
-            ipa = {}
-            apks.append(ipa)
 
-            ipa["apkName"] = os.path.basename(f)
+    ipa = {
-            ipa["hash"] = common.sha256sum(f)
+        "apkName": os.path.basename(ipa_path),
-            ipa["hashType"] = "sha256"
+        "hash": sha256,
-            ipa["size"] = os.path.getsize(f)
+        "hashType": "sha256",
+        "size": file_size,
+    }
 
-            with zipfile.ZipFile(f) as ipa_zip:
+    with zipfile.ZipFile(ipa_path) as ipa_zip:
         for info in ipa_zip.infolist():
             if re.match("Payload/[^/]*.app/Info.plist", info.filename):
                 with ipa_zip.open(info) as plist_file:
@@ -576,6 +559,53 @@ def process_ipa(repodir, apks):
                     ipa["versionCode"] = version_string_to_int(plist["CFBundleShortVersionString"])
                     ipa["versionName"] = plist["CFBundleShortVersionString"]
                     ipa["usage"] = {k: v for k, v in plist.items() if 'Usage' in k}
+    return ipa
+
+
+def scan_repo_for_ipas(apkcache, repodir, knownapks):
+    """Scan for IPA files in a given repo directory.
+
+    Parameters
+    ----------
+    apkcache
+      cache dictionary containting cached file infos from previous runs
+    repodir
+      repo directory to scan
+    knownapks
+      list of all known files, as per metadata.read_metadata
+
+    Returns
+    -------
+    ipas
+      list of file infos for ipa files in ./repo folder
+    cachechanged
+      ture if new ipa files were found and added to `apkcache`
+    """
+    cachechanged = False
+    ipas = []
+    for ipa_path in glob.glob(os.path.join(repodir, '*.ipa')):
+        ipa_name = os.path.basename(ipa_path)
+
+        file_size = os.stat(ipa_path).st_size
+        if file_size == 0:
+            raise FDroidException(_('{path} is zero size!')
+                                  .format(path=ipa_path))
+
+        sha256 = common.sha256sum(ipa_path)
+        ipa = apkcache.get(ipa_name, {})
+
+        if ipa.get('hash') != sha256:
+            ipa = parse_ipa(ipa_path, file_size, sha256)
+            apkcache[ipa_name] = ipa
+            cachechanged = True
+
+        added = knownapks.recordapk(ipa_name, ipa['packageName'])
+        if added:
+            ipa['added'] = added
+
+        ipas.append(ipa)
+
+    return ipas, cachechanged
 
 
 def translate_per_build_anti_features(apps, apks):
@@ -1175,7 +1205,10 @@ def insert_localized_app_metadata(apps):
 
 
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
-    """Scan a repo for all files with an extension except APK/OBB.
+    """Scan a repo for all files with an extension except APK/OBB/IPA.
+
+    This allows putting all kinds of files into repostories. E.g. Media Files,
+    Zip archives, ...
 
     Parameters
     ----------
@@ -1192,22 +1225,29 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     repo_files = []
     repodir = repodir.encode()
     for name in os.listdir(repodir):
+        # skip files based on file extensions, that are handled elsewhere
         file_extension = common.get_file_extension(name)
         if file_extension in ('apk', 'obb', 'ipa'):
             continue
+
+        # skip source tarballs generated by fdroidserver
         filename = os.path.join(repodir, name)
         name_utf8 = name.decode()
         if filename.endswith(b'_src.tar.gz'):
             logging.debug(_('skipping source tarball: {path}')
                           .format(path=filename.decode()))
             continue
+
+        # skip all other files generated by fdroidserver
         if not common.is_repo_file(filename):
             continue
+
         stat = os.stat(filename)
         if stat.st_size == 0:
             raise FDroidException(_('{path} is zero size!')
                                   .format(path=filename))
 
+        # load file infos from cache if not stale
         shasum = common.sha256sum(filename)
         usecache = False
         if name_utf8 in apkcache:
@@ -1220,6 +1260,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
                 logging.debug(_("Ignoring stale cache data for {apkfilename}")
                               .format(apkfilename=name_utf8))
 
+        # scan file if info wasn't in cache
         if not usecache:
             logging.debug(_("Processing {apkfilename}").format(apkfilename=name_utf8))
             repo_file = collections.OrderedDict()
@@ -2182,7 +2223,6 @@ def prepare_apps(apps, apks, repodir):
     -------
     the relevant subset of apps (as a deepcopy)
     """
-    process_ipa(repodir, apks)
     apps_with_packages = get_apps_with_packages(apps, apks)
     apply_info_from_latest_apk(apps_with_packages, apks)
     insert_funding_yml_donation_links(apps)
@@ -2309,6 +2349,11 @@ def main():
                                            options.use_date_from_apk)
     cachechanged = cachechanged or fcachechanged
     apks += files
+
+    ipas, icachechanged = scan_repo_for_ipas(apkcache, repodirs[0], knownapks)
+    cachechanged = cachechanged or icachechanged
+    apks += ipas
+
     appid_has_apks = set()
     appid_has_repo_files = set()
     remove_apks = []