Merge branch 'ios-metadata' into 'master'

draft: support iOS fastlane metadata

See merge request fdroid/fdroidserver!1426

.gitlab-ci.yml

@@ -542,7 +542,7 @@ servergitmirrors:
     - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
     - cd /tmp/fdroid
     - touch fdroid-icon.png
-    - printf "servergitmirrors:\n-\ url:\ $SERVER_GIT_MIRROR\n" >> config.yml
+    - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
     - $PYTHONPATH/fdroid update --verbose --create-metadata
     - $PYTHONPATH/fdroid deploy --verbose
     - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`

examples/config.yml

@@ -203,6 +203,10 @@
 #
 # servergitmirrors: https://github.com/user/repo
 # servergitmirrors:
+#   - https://github.com/user/repo
+#   - https://gitlab.com/user/repo
+#
+# servergitmirrors:
 #   - url: https://github.com/user/repo
 #   - url: https://gitlab.com/user/repo
 #     indexOnly: true

fdroidserver/update.py

@@ -34,6 +34,7 @@ import json
 import time
 import yaml
 import copy
+import pathlib
 import defusedxml.ElementTree as ElementTree
 from datetime import datetime, timezone
 from argparse import ArgumentParser
@@ -1136,6 +1137,9 @@ def insert_localized_app_metadata(apps):
                         if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict):
                             apps[packageName][base] = collections.OrderedDict()
                         apps[packageName][base][locale] = common.file_entry(dst)
+
+            # copy screenshots from local source code checkout into wellknown
+            # location in repo directory
             for d in dirs:
                 if d in SCREENSHOT_DIRS:
                     if locale == 'images':
@@ -1148,6 +1152,8 @@ def insert_localized_app_metadata(apps):
                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
                             _strip_and_copy_image(f, screenshotdestdir)
 
+
+def ingest_screenshots_from_repo_dir(apps):
     repodirs = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z]*')))
     for d in repodirs:
         if not os.path.isdir(d):
@@ -1208,6 +1214,86 @@ def insert_localized_app_metadata(apps):
                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 
 
+LANG_CODE = re.compile(r'^[a-z]{2}([-_][A-Z][a-zA-Z]{1,3})?$')
+
+
+FASTLANE_IOS_MAP = {
+    "name.txt": 'name',
+    "subtitle.txt": 'summary',
+    "description.txt": 'description',
+}
+
+
+def parse_ios_screenshot_name(path):
+    """
+    Infer type info from screenshot file name.
+
+    Device type/name is part of the file name of iOS fastlane screenshots.
+    Here are some example:
+     * 'iPhone 8+ @ iOS 16-1.png'
+     * 'iPad Pro 12.9" 2gen @ iOS 16-1.png'
+     * '1_ipadPro129_1.1.png'
+     * '1_iphone6Plus_1.1.png'
+    """
+    s = path.stem.split('@')
+    if len(s) >= 2:
+        if "iphone" in s[0].lower():
+            return ("phoneScreenshots", '@'.join(s[1:]))
+        elif "ipad" in s[0].lower():
+            return ("tenInchScreenshots", "@".join(s[1:]))
+    else:
+        return ('phoneScreenshots', s[0])
+
+
+def insert_localized_ios_app_metadata(apps_with_packages):
+
+    if not any(pathlib.Path('repo').glob('*.ipa')):
+        # no IPA files present in repo, nothing to do here, exiting early
+        return
+
+    for package_name, app in apps_with_packages.items():
+        if not any(pathlib.Path('repo').glob(f'{package_name}*.ipa')):
+            # couldn't find any IPA files for this package_name
+            # so we don't have to look for fastlane data
+            continue
+
+        fastlane_dir = pathlib.Path('build', package_name, 'fastlane')
+
+        for lang_dir in (fastlane_dir / 'metadata').iterdir():
+            lang_code = lang_dir.name
+            m = LANG_CODE.match(lang_code)
+            if m:
+                for metadata_file in (lang_dir).iterdir():
+                    key = FASTLANE_IOS_MAP.get(metadata_file.name)
+                    if key:
+                        _set_localized_text_entry(app, lang_code, key, metadata_file)
+
+        # discover available screenshots and put findings in a dict
+        screenshots = {}
+        for lang_sdir in (fastlane_dir / 'screenshots').iterdir():
+            lang_code = lang_sdir.name
+            m = LANG_CODE.match(lang_code)
+            if m:
+                screenshots[lang_code] = {}
+                for screenshot in (lang_sdir).iterdir():
+                    if screenshot.suffix[1:] in ALLOWED_EXTENSIONS:
+                        # asdf #TODO
+                        device_name, screenshot_name = parse_ios_screenshot_name(screenshot)
+
+                        if not screenshots[lang_code].get(device_name):
+                            screenshots[lang_code][device_name] = {}
+                        screenshots[lang_code][device_name][screenshot_name] = screenshot
+
+        # copy screenshots to repo dir
+        for lang_code in screenshots.keys():
+            for device in screenshots[lang_code].keys():
+                dest_dir = pathlib.Path('repo') / package_name / lang_code / device
+                dest_dir.mkdir(mode=0o755, parents=True, exist_ok=True)
+                for name, path in screenshots[lang_code][device].items():
+                    dest = dest_dir / (name.replace(" ", "_").replace("\t", "_") + path.suffix)
+                    _strip_and_copy_image(str(path), str(dest))
+
+
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     """Scan a repo for all files with an extension except APK/OBB/IPA.
 
@@ -2238,6 +2324,8 @@ def prepare_apps(apps, apks, repodir):
     translate_per_build_anti_features(apps_with_packages, apks)
     if repodir == 'repo':
         insert_localized_app_metadata(apps_with_packages)
+        insert_localized_ios_app_metadata(apps_with_packages)
+        ingest_screenshots_from_repo_dir(apps_with_packages)
     insert_missing_app_names_from_apks(apps_with_packages, apks)
     return apps_with_packages
 

gradlew-fdroid

@@ -187,6 +187,7 @@ get_sha() {
         '7.6.1')  echo '6147605a23b4eff6c334927a86ff3508cb5d6722cd624c97ded4c2e8640f1f87' ;;
         '7.6.2')  echo 'a01b6587e15fe7ed120a0ee299c25982a1eee045abd6a9dd5e216b2f628ef9ac' ;;
         '7.6.3')  echo '740c2e472ee4326c33bf75a5c9f5cd1e69ecf3f9b580f6e236c86d1f3d98cfac' ;;
+        '7.6.4')  echo 'bed1da33cca0f557ab13691c77f38bb67388119e4794d113e051039b80af9bb1' ;;
         '8.0')    echo '4159b938ec734a8388ce03f52aa8f3c7ed0d31f5438622545de4f83a89b79788' ;;
         '8.0.1')  echo '1b6b558be93f29438d3df94b7dfee02e794b94d9aca4611a92cdb79b6b88e909' ;;
         '8.0.2')  echo 'ff7bf6a86f09b9b2c40bb8f48b25fc19cf2b2664fd1d220cd7ab833ec758d0d7' ;;
@@ -218,7 +219,7 @@ d_gradle_plugin_ver_k=(8.4 8.3 8.2 8.1 8.0 7.4 7.3 7.2.0 7.1 7.0 4.2 4.1 4.0 3.6
 d_plugin_min_gradle_v=(8.6 8.4 8.2 8.0 8.0 7.5 7.4 7.3.3 7.2 7.0.2 6.7.1 6.5 6.1.1 5.6.4 5.4.1 5.1.1 4.10.1 4.6 4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
 
 # All gradle versions we know about
-plugin_v=(8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
+plugin_v=(8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
 
 v_all=${plugin_v[@]}
 

tests/update.TestCase

@@ -165,6 +165,7 @@ class UpdateTest(unittest.TestCase):
         apps['eu.siacs.conversations']['Builds'] = [build_conversations]
 
         fdroidserver.update.insert_localized_app_metadata(apps)
+        fdroidserver.update.ingest_screenshots_from_repo_dir(apps)
 
         appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US')
         self.assertTrue(
@@ -278,6 +279,7 @@ class UpdateTest(unittest.TestCase):
         knownapks = fdroidserver.common.KnownApks()
         apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False)
         fdroidserver.update.insert_localized_app_metadata(apps)
+        fdroidserver.update.ingest_screenshots_from_repo_dir(apps)
         fdroidserver.update.apply_info_from_latest_apk(apps, apks)
         app = apps['info.guardianproject.urzip']
         self.assertIsNone(app.Name)