Merge branch 'index_v2' into 'master'

Index v2

See merge request fdroid/fdroidserver!1092

.gitlab-ci.yml

@@ -9,6 +9,7 @@ variables:
 ci-images-base run-tests:
   image: registry.gitlab.com/fdroid/ci-images-base
   script:
+    - $pip install --upgrade pip setuptools wheel  # make this go away: "error: invalid command 'bdist_wheel'"
     - $pip install -e .
     - ./tests/run-tests
     # make sure that translations do not cause stacktraces

fdroidserver/common.py

@@ -4088,26 +4088,33 @@ def get_per_app_repos():
     return repos
 
 
-def is_repo_file(filename):
+def is_repo_file(filename, for_gpg_signing=False):
     """Whether the file in a repo is a build product to be delivered to users."""
     if isinstance(filename, str):
         filename = filename.encode('utf-8', errors="surrogateescape")
-    return os.path.isfile(filename) \
+    ignore_files = [
-        and not filename.endswith(b'.asc') \
-        and not filename.endswith(b'.sig') \
-        and not filename.endswith(b'.idsig') \
-        and not filename.endswith(b'.log.gz') \
-        and os.path.basename(filename) not in [
-            b'index.css',
-            b'index.jar',
-            b'index_unsigned.jar',
-            b'index.xml',
-            b'index.html',
-            b'index.png',
-            b'index-v1.jar',
-            b'index-v1.json',
         b'categories.txt',
+        b'entry.jar',
+        b'index-v1.jar',
+        b'index-v2.jar',
+        b'index.css',
+        b'index.html',
+        b'index.jar',
+        b'index.png',
+        b'index.xml',
+        b'index_unsigned.jar',
     ]
+    if not for_gpg_signing:
+        ignore_files += [b'entry.json', b'index-v1.json', b'index-v2.json']
+
+    return (
+        os.path.isfile(filename)
+        and not filename.endswith(b'.asc')
+        and not filename.endswith(b'.sig')
+        and not filename.endswith(b'.idsig')
+        and not filename.endswith(b'.log.gz')
+        and os.path.basename(filename) not in ignore_files
+    )
 
 
 def get_examples_dir():

fdroidserver/gpgsign.py

@@ -65,9 +65,7 @@ def main():
 
         # Process any apks that are waiting to be signed...
         for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))):
-            if common.get_file_extension(f) == 'asc':
+            if not common.is_repo_file(f, for_gpg_signing=True):
-                continue
-            if not common.is_repo_file(f) and not f.endswith('/index-v1.json'):
                 continue
             filename = os.path.basename(f)
             sigfilename = filename + ".asc"

fdroidserver/index.py

@@ -28,11 +28,13 @@ import re
 import shutil
 import tempfile
 import urllib.parse
+import yaml
 import zipfile
 import calendar
 import qrcode
 from binascii import hexlify, unhexlify
 from datetime import datetime, timezone
+from pathlib import Path
 from xml.dom.minidom import Document
 
 from . import _
@@ -136,6 +138,8 @@ def make(apps, apks, repodir, archive):
             fdroid_signing_key_fingerprints)
     make_v1(sortedapps, apks, repodir, repodict, requestsdict,
             fdroid_signing_key_fingerprints)
+    make_v2(sortedapps, apks, repodir, repodict, requestsdict,
+            fdroid_signing_key_fingerprints, archive)
     make_website(sortedapps, repodir, repodict)
 
 
@@ -469,6 +473,393 @@ fieldset select, fieldset input, #reposelect select, #reposelect input {
 }""")
 
 
+def dict_diff(source, target):
+    if not isinstance(target, dict) or not isinstance(source, dict):
+        return target
+
+    result = {key: None for key in source if key not in target}
+
+    for key, value in target.items():
+        if key not in source:
+            result[key] = value
+        elif value != source[key]:
+            result[key] = dict_diff(source[key], value)
+
+    return result
+
+
+def file_entry(filename, hashType=None, hsh=None, size=None):
+    meta = {}
+    meta["name"] = "/" + filename.split("/", 1)[1]
+    if hsh:
+        meta[hashType] = hsh
+    if hsh != "sha256":
+        meta["sha256"] = common.sha256sum(filename)
+    if size:
+        meta["size"] = size
+    else:
+        meta["size"] = os.stat(filename).st_size
+    return meta
+
+
+def load_locale(name, repodir):
+    lst = {}
+    for yml in Path().glob("config/**/{name}.yml".format(name=name)):
+        locale = yml.parts[1]
+        if len(yml.parts) == 2:
+            locale = "en-US"
+        with open(yml, encoding="utf-8") as fp:
+            elem = yaml.safe_load(fp)
+            for akey, avalue in elem.items():
+                if akey not in lst:
+                    lst[akey] = {}
+                for key, value in avalue.items():
+                    if key not in lst[akey]:
+                        lst[akey][key] = {}
+                    if key == "icon":
+                        shutil.copy(os.path.join("config", value), os.path.join(repodir, "icons"))
+                        lst[akey][key][locale] = file_entry(os.path.join(repodir, "icons", value))
+                    else:
+                        lst[akey][key][locale] = value
+
+    return lst
+
+
+def convert_datetime(obj):
+    if isinstance(obj, datetime):
+        # Java prefers milliseconds
+        # we also need to account for time zone/daylight saving time
+        return int(calendar.timegm(obj.timetuple()) * 1000)
+    return obj
+
+
+def package_metadata(app, repodir):
+    meta = {}
+    for element in (
+        "added",
+        # "binaries",
+        "Categories",
+        "Changelog",
+        "IssueTracker",
+        "lastUpdated",
+        "License",
+        "SourceCode",
+        "Translation",
+        "WebSite",
+        "video",
+        "featureGraphic",
+        "promoGraphic",
+        "tvBanner",
+        "screenshots",
+        "AuthorEmail",
+        "AuthorName",
+        "AuthorPhone",
+        "AuthorWebSite",
+        "Bitcoin",
+        "FlattrID",
+        "Liberapay",
+        "LiberapayID",
+        "Litecoin",
+        "OpenCollective",
+    ):
+        if element in app and app[element]:
+            element_new = element[:1].lower() + element[1:]
+            meta[element_new] = convert_datetime(app[element])
+
+    for element in (
+        "Name",
+        "Summary",
+        "Description",
+    ):
+        element_new = element[:1].lower() + element[1:]
+        if element in app and app[element]:
+            meta[element_new] = {"en-US": convert_datetime(app[element])}
+        elif "localized" in app:
+            localized = {k: v[element_new] for k, v in app["localized"].items() if element_new in v}
+            if localized:
+                meta[element_new] = localized
+
+    if "name" not in meta and app["AutoName"]:
+        meta["name"] = {"en-US": app["AutoName"]}
+
+    # fdroidserver/metadata.py App default
+    if meta["license"] == "Unknown":
+        del meta["license"]
+
+    if app["Donate"]:
+        meta["donate"] = [app["Donate"]]
+
+    # TODO handle different resolutions
+    if app.get("icon"):
+        meta["icon"] = {"en-US": file_entry(os.path.join(repodir, "icons", app["icon"]))}
+
+    if "iconv2" in app:
+        meta["icon"] = app["iconv2"]
+
+    return meta
+
+
+def convert_version(version, app, repodir):
+    ver = {}
+    if "added" in version:
+        ver["added"] = convert_datetime(version["added"])
+    else:
+        ver["added"] = 0
+
+    ver["file"] = {
+        "name": "/{}".format(version["apkName"]),
+        version["hashType"]: version["hash"],
+        "size": version["size"]
+    }
+
+    if "srcname" in version:
+        ver["src"] = file_entry(os.path.join(repodir, version["srcname"]))
+
+    if "obbMainFile" in version:
+        ver["obbMainFile"] = file_entry(
+            os.path.join(repodir, version["obbMainFile"]),
+            "sha256", version["obbMainFileSha256"]
+        )
+
+    if "obbPatchFile" in version:
+        ver["obbPatchFile"] = file_entry(
+            os.path.join(repodir, version["obbPatchFile"]),
+            "sha256", version["obbPatchFileSha256"]
+        )
+
+    ver["manifest"] = manifest = {}
+
+    for element in (
+        "nativecode",
+        "versionName",
+        "maxSdkVersion",
+    ):
+        if element in version:
+            manifest[element] = version[element]
+
+    if "versionCode" in version:
+        manifest["versionCode"] = int(version["versionCode"])
+
+    if "features" in version and version["features"]:
+        manifest["features"] = features = []
+        for feature in version["features"]:
+            # TODO get version from manifest, default (0) is omitted
+            # features.append({"name": feature, "version": 1})
+            features.append({"name": feature})
+
+    if "minSdkVersion" in version:
+        manifest["usesSdk"] = {}
+        manifest["usesSdk"]["minSdkVersion"] = version["minSdkVersion"]
+        if "targetSdkVersion" in version:
+            manifest["usesSdk"]["targetSdkVersion"] = version["targetSdkVersion"]
+        else:
+            # https://developer.android.com/guide/topics/manifest/uses-sdk-element.html#target
+            manifest["usesSdk"]["targetSdkVersion"] = manifest["usesSdk"]["minSdkVersion"]
+
+    if "signer" in version:
+        manifest["signer"] = {"sha256": [version["signer"]]}
+
+    for element in ("uses-permission", "uses-permission-sdk-23"):
+        en = element.replace("uses-permission", "usesPermission").replace("-sdk-23", "Sdk23")
+        if element in version and version[element]:
+            manifest[en] = []
+            for perm in version[element]:
+                if perm[1]:
+                    manifest[en].append({"name": perm[0], "maxSdkVersion": perm[1]})
+                else:
+                    manifest[en].append({"name": perm[0]})
+
+    if "AntiFeatures" in app and app["AntiFeatures"]:
+        ver["antiFeatures"] = {}
+        for antif in app["AntiFeatures"]:
+            # TODO: get reasons from fdroiddata
+            # ver["antiFeatures"][antif] = {"en-US": "reason"}
+            ver["antiFeatures"][antif] = {}
+
+    if "AntiFeatures" in version and version["AntiFeatures"]:
+        if "antiFeatures" not in ver:
+            ver["antiFeatures"] = {}
+        for antif in version["AntiFeatures"]:
+            # TODO: get reasons from fdroiddata
+            # ver["antiFeatures"][antif] = {"en-US": "reason"}
+            ver["antiFeatures"][antif] = {}
+
+    if "versionCode" in version:
+        if int(version["versionCode"]) > int(app["CurrentVersionCode"]):
+            ver["releaseChannels"] = ["Beta"]
+
+    versionCodeStr = str(version['versionCode'])  # TODO build.versionCode should be int!
+    for build in app.get('Builds', []):
+        if build['versionCode'] == versionCodeStr and "whatsNew" in build:
+            ver["whatsNew"] = build["whatsNew"]
+            break
+
+    return ver
+
+
+def v2_repo(repodict, repodir, archive):
+    repo = {}
+
+    repo["name"] = {"en-US": repodict["name"]}
+    repo["description"] = {"en-US": repodict["description"]}
+    repo["icon"] = {"en-US": file_entry("{}/icons/{}".format(repodir, repodict["icon"]))}
+
+    config = load_locale("config", repodir)
+    if config:
+        repo["name"] = config["archive" if archive else "repo"]["name"]
+        repo["description"] = config["archive" if archive else "repo"]["description"]
+        repo["icon"] = config["archive" if archive else "repo"]["icon"]
+
+    repo["address"] = repodict["address"]
+    repo["webBaseUrl"] = "https://f-droid.org/packages/"
+
+    if "repo_url" in common.config:
+        primary_mirror = common.config["repo_url"][:-len("/repo")]
+        if "mirrors" in repodict and primary_mirror not in repodict["mirrors"]:
+            repodict["mirrors"].append(primary_mirror)
+
+    if "mirrors" in repodict:
+        repo["mirrors"] = [{"url": mirror} for mirror in repodict["mirrors"]]
+
+    repo["timestamp"] = repodict["timestamp"]
+
+    anti_features = load_locale("antiFeatures", repodir)
+    if anti_features:
+        repo["antiFeatures"] = anti_features
+
+    categories = load_locale("categories", repodir)
+    if categories:
+        repo["categories"] = categories
+
+    channels = load_locale("channels", repodir)
+    if channels:
+        repo["releaseChannels"] = channels
+
+    return repo
+
+
+def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints, archive):
+
+    def _index_encoder_default(obj):
+        if isinstance(obj, set):
+            return sorted(list(obj))
+        if isinstance(obj, datetime):
+            # Java prefers milliseconds
+            # we also need to account for time zone/daylight saving time
+            return int(calendar.timegm(obj.timetuple()) * 1000)
+        if isinstance(obj, dict):
+            d = collections.OrderedDict()
+            for key in sorted(obj.keys()):
+                d[key] = obj[key]
+            return d
+        raise TypeError(repr(obj) + " is not JSON serializable")
+
+    output = collections.OrderedDict()
+    output["repo"] = v2_repo(repodict, repodir, archive)
+    if requestsdict and requestsdict["install"] or requestsdict["uninstall"]:
+        output["repo"]["requests"] = requestsdict
+
+    # establish sort order of the index
+    v1_sort_packages(packages, fdroid_signing_key_fingerprints)
+
+    output_packages = collections.OrderedDict()
+    output['packages'] = output_packages
+    for package in packages:
+        packageName = package['packageName']
+        if packageName not in apps:
+            logging.info(_('Ignoring package without metadata: ') + package['apkName'])
+            continue
+        if not package.get('versionName'):
+            app = apps[packageName]
+            versionCodeStr = str(package['versionCode'])  # TODO build.versionCode should be int!
+            for build in app.get('Builds', []):
+                if build['versionCode'] == versionCodeStr:
+                    versionName = build.get('versionName')
+                    logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}')
+                                 .format(apkfilename=package['apkName'], version=versionName))
+                    package['versionName'] = versionName
+                    break
+        if packageName in output_packages:
+            packagelist = output_packages[packageName]
+        else:
+            packagelist = {}
+            output_packages[packageName] = packagelist
+            packagelist["metadata"] = package_metadata(apps[packageName], repodir)
+            if "signer" in package:
+                packagelist["metadata"]["preferredSigner"] = package["signer"]
+
+            packagelist["versions"] = {}
+
+        packagelist["versions"][package["hash"]] = convert_version(package, apps[packageName], repodir)
+
+    entry = {}
+    entry["timestamp"] = repodict["timestamp"]
+
+    entry["version"] = repodict["version"]
+    if "maxage" in repodict:
+        entry["maxAge"] = repodict["maxage"]
+
+    json_name = 'index-v2.json'
+    index_file = os.path.join(repodir, json_name)
+    with open(index_file, "w", encoding="utf-8") as fp:
+        if common.options.pretty:
+            json.dump(output, fp, default=_index_encoder_default, indent=2, ensure_ascii=False)
+        else:
+            json.dump(output, fp, default=_index_encoder_default, ensure_ascii=False)
+
+    json_name = "tmp/{}_{}.json".format(repodir, convert_datetime(repodict["timestamp"]))
+    with open(json_name, "w", encoding="utf-8") as fp:
+        if common.options.pretty:
+            json.dump(output, fp, default=_index_encoder_default, indent=2, ensure_ascii=False)
+        else:
+            json.dump(output, fp, default=_index_encoder_default, ensure_ascii=False)
+
+    entry["index"] = file_entry(index_file)
+    entry["index"]["numPackages"] = len(output.get("packages", []))
+
+    indexes = sorted(Path().glob("tmp/{}*.json".format(repodir)), key=lambda x: x.name)
+    indexes.pop()  # remove current index
+    # remove older indexes
+    while len(indexes) > 10:
+        indexes.pop(0).unlink()
+
+    indexes = [json.loads(Path(fn).read_text(encoding="utf-8")) for fn in indexes]
+
+    for diff in Path().glob("{}/diff/*.json".format(repodir)):
+        diff.unlink()
+
+    entry["diffs"] = {}
+    for old in indexes:
+        diff_name = str(old["repo"]["timestamp"]) + ".json"
+        diff_file = os.path.join(repodir, "diff", diff_name)
+        diff = dict_diff(old, output)
+        if not os.path.exists(os.path.join(repodir, "diff")):
+            os.makedirs(os.path.join(repodir, "diff"))
+        with open(diff_file, "w", encoding="utf-8") as fp:
+            if common.options.pretty:
+                json.dump(diff, fp, default=_index_encoder_default, indent=2, ensure_ascii=False)
+            else:
+                json.dump(diff, fp, default=_index_encoder_default, ensure_ascii=False)
+
+        entry["diffs"][old["repo"]["timestamp"]] = file_entry(diff_file)
+        entry["diffs"][old["repo"]["timestamp"]]["numPackages"] = len(diff.get("packages", []))
+
+    json_name = "entry.json"
+    index_file = os.path.join(repodir, json_name)
+    with open(index_file, "w", encoding="utf-8") as fp:
+        if common.options.pretty:
+            json.dump(entry, fp, default=_index_encoder_default, indent=2, ensure_ascii=False)
+        else:
+            json.dump(entry, fp, default=_index_encoder_default, ensure_ascii=False)
+
+    if common.options.nosign:
+        _copy_to_local_copy_dir(repodir, index_file)
+        logging.debug(_('index-v2 must have a signature, use `fdroid signindex` to create it!'))
+    else:
+        signindex.config = common.config
+        signindex.sign_index(repodir, json_name, signindex.HashAlg.SHA256)
+
+
 def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints):
 
     def _index_encoder_default(obj):
@@ -504,7 +895,10 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
                      'ArchivePolicy', 'AutoName', 'AutoUpdateMode', 'MaintainerNotes',
                      'Provides', 'Repo', 'RepoType', 'RequiresRoot',
                      'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
-                     'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
+                     'UpdateCheckName', 'NoSourceSince', 'VercodeOperation',
+                     'summary', 'description', 'promoGraphic', 'screenshots', 'whatsNew',
+                     'featureGraphic', 'iconv2', 'tvBanner',
+                     ):
                 continue
 
             # name things after the App class fields in fdroidclient
@@ -573,7 +967,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
         logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
     else:
         signindex.config = common.config
-        signindex.sign_index_v1(repodir, json_name)
+        signindex.sign_index(repodir, json_name)
 
 
 def _copy_to_local_copy_dir(repodir, f):

fdroidserver/signindex.py

@@ -21,6 +21,7 @@ import os
 import time
 import zipfile
 from argparse import ArgumentParser
+from enum import Enum
 import logging
 
 from . import _
@@ -33,16 +34,33 @@ options = None
 start_timestamp = time.gmtime()
 
 
-def sign_jar(jar):
+HashAlg = Enum("SHA1", "SHA256")
+
+
+def sign_jar(jar, hash_algorithm=None):
     """Sign a JAR file with Java's jarsigner.
 
     This method requires a properly initialized config object.
-
-    This does use old hashing algorithms, i.e. SHA1, but that's not
-    broken yet for file verification.  This could be set to SHA256,
-    but then Android < 4.3 would not be able to verify it.
-    https://code.google.com/p/android/issues/detail?id=38321
     """
+    if hash_algorithm == HashAlg.SHA256:
+        args = [
+            config['jarsigner'],
+            '-keystore',
+            config['keystore'],
+            '-storepass:env',
+            'FDROID_KEY_STORE_PASS',
+            '-digestalg',
+            'SHA-256',
+            '-sigalg',
+            'SHA256withRSA',
+            jar,
+            config['repo_keyalias'],
+        ]
+    else:
+        # This does use old hashing algorithms, i.e. SHA1, but that's not
+        # broken yet for file verification.  This could be set to SHA256,
+        # but then Android < 4.3 would not be able to verify it.
+        # https://code.google.com/p/android/issues/detail?id=38321
         args = [
             config['jarsigner'],
             '-keystore',
@@ -69,7 +87,7 @@ def sign_jar(jar):
         raise FDroidException("Failed to sign %s!" % jar)
 
 
-def sign_index_v1(repodir, json_name):
+def sign_index(repodir, json_name, hash_algorithm=None):
     """Sign index-v1.json to make index-v1.jar.
 
     This is a bit different than index.jar: instead of their being index.xml
@@ -84,12 +102,14 @@ def sign_index_v1(repodir, json_name):
     # Test if index is valid
     with open(index_file, encoding="utf-8") as fp:
         index = json.load(fp)
+        # TODO: add test for index-v2
+        if "apps" in index:
             [metadata.App(app) for app in index["apps"]]
 
     jar_file = os.path.join(repodir, name + '.jar')
     with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
         jar.write(index_file, json_name)
-    sign_jar(jar_file)
+    sign_jar(jar_file, hash_algorithm)
 
 
 def status_update_json(signed):
@@ -138,7 +158,14 @@ def main():
         json_name = 'index-v1.json'
         index_file = os.path.join(output_dir, json_name)
         if os.path.exists(index_file):
-            sign_index_v1(output_dir, json_name)
+            sign_index(output_dir, json_name)
+            logging.info('Signed ' + index_file)
+            signed.append(index_file)
+
+        json_name = 'entry.json'
+        index_file = os.path.join(output_dir, json_name)
+        if os.path.exists(index_file):
+            sign_index(output_dir, json_name, HashAlg.SHA256)
             logging.info('Signed ' + index_file)
             signed.append(index_file)
 

fdroidserver/update.py

@@ -554,7 +554,7 @@ def _get_localized_dict(app, locale):
     return app['localized'][locale]
 
 
-def _set_localized_text_entry(app, locale, key, f):
+def _set_localized_text_entry(app, locale, key, f, versionCode=None):
     """Read a fastlane/triple-t metadata file and add an entry to the app.
 
     This reads more than the limit, in case there is leading or
@@ -563,9 +563,17 @@ def _set_localized_text_entry(app, locale, key, f):
     """
     try:
         limit = config['char_limits'][key]
+        if not versionCode:
             localized = _get_localized_dict(app, locale)
         with open(f, errors='replace') as fp:
             text = fp.read(limit * 2)
+            if versionCode:
+                for build in app["Builds"]:
+                    if int(build["versionCode"]) == versionCode:
+                        if "whatsNew" not in build:
+                            build["whatsNew"] = collections.OrderedDict()
+                        build["whatsNew"][locale] = text[:limit]
+                        return
             if len(text) > 0:
                 if key in ('name', 'summary', 'video'):  # hardcoded as a single line
                     localized[key] = text.strip('\n')[:limit]
@@ -987,15 +995,33 @@ def insert_localized_app_metadata(apps):
                     locale = segments[-2]
                     _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
                                               os.path.join(root, f))
-                    continue
 
                 base, extension = common.get_extension(f)
+
+                if extension == 'txt':
+                    try:
+                        versionCode = int(base)
+                        locale = segments[-2]
+                        if base in [a["versionCode"] for a in apps[packageName]["Builds"]]:
+                            _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
+                                                      os.path.join(root, f), versionCode)
+                        continue
+                    except ValueError:
+                        pass
+
                 if locale == 'images':
                     locale = segments[-2]
                     destdir = os.path.join('repo', packageName, locale)
                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
                     os.makedirs(destdir, mode=0o755, exist_ok=True)
                     _strip_and_copy_image(os.path.join(root, f), destdir)
+                    dst = os.path.join(destdir, f)
+                    if os.path.isfile(dst):
+                        if base == "icon":
+                            base = "iconv2"
+                        if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict):
+                            apps[packageName][base] = collections.OrderedDict()
+                        apps[packageName][base][locale] = index.file_entry(dst)
             for d in dirs:
                 if d in SCREENSHOT_DIRS:
                     if locale == 'images':
@@ -1044,12 +1070,26 @@ def insert_localized_app_metadata(apps):
                     if not os.path.exists(index_file):
                         os.link(f, index_file, follow_symlinks=False)
                     graphics[base] = filename
+                    if base == "icon":
+                        base = "iconv2"
+                    if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict):
+                        apps[packageName][base] = collections.OrderedDict()
+                    apps[packageName][base][locale] = index.file_entry(index_file)
             elif screenshotdir in SCREENSHOT_DIRS:
                 # there can any number of these per locale
                 logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f))
                 if screenshotdir not in graphics:
                     graphics[screenshotdir] = []
                 graphics[screenshotdir].append(filename)
+
+                newKey = screenshotdir.replace("Screenshots", "")
+                if "screenshots" not in apps[packageName]:
+                    apps[packageName]["screenshots"] = collections.OrderedDict()
+                if newKey not in apps[packageName]["screenshots"]:
+                    apps[packageName]["screenshots"][newKey] = collections.OrderedDict()
+                if locale not in apps[packageName]["screenshots"][newKey]:
+                    apps[packageName]["screenshots"][newKey][locale] = []
+                apps[packageName]["screenshots"][newKey][locale].append(index.file_entry(f))
             else:
                 logging.warning(_('Unsupported graphics file found: {path}').format(path=f))
 

tests/common.TestCase

@@ -2372,6 +2372,72 @@ class CommonTest(unittest.TestCase):
         fdroidserver.common.set_FDroidPopen_env(build)
         self.assertNotIn('', os.getenv('PATH').split(os.pathsep))
 
+    def test_is_repo_file(self):
+        is_repo_file = fdroidserver.common.is_repo_file
+        self.assertFalse(is_repo_file('does-not-exist'))
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            Path('repo').mkdir()
+            repo_files = [
+                'repo/com.example.test.helloworld_1.apk',
+                'repo/com.politedroid_6.apk',
+                'repo/duplicate.permisssions_9999999.apk',
+                'repo/fake.ota.update_1234.zip',
+                'repo/info.guardianproject.index-v1.jar_123.apk',
+                'repo/info.zwanenburg.caffeinetile_4.apk',
+                'repo/main.1101613.obb.main.twoversions.obb',
+            ]
+            index_files = [
+                'repo/entry.jar',
+                'repo/entry.json',
+                'repo/index-v1.jar',
+                'repo/index-v1.json',
+                'repo/index-v2.jar',
+                'repo/index-v2.json',
+                'repo/index.css',
+                'repo/index.html',
+                'repo/index.jar',
+                'repo/index.png',
+                'repo/index.xml',
+            ]
+            non_repo_files = ['repo/categories.txt']
+            for f in repo_files + index_files + non_repo_files:
+                open(f, 'w').close()
+
+            repo_dirs = [
+                'repo/com.politedroid',
+                'repo/info.guardianproject.index-v1.jar',
+                'repo/status',
+            ]
+            for d in repo_dirs:
+                os.mkdir(d)
+
+            for f in repo_files:
+                self.assertTrue(os.path.exists(f), f + ' was created')
+                self.assertTrue(is_repo_file(f), f + ' is repo file')
+
+            for f in index_files:
+                self.assertTrue(os.path.exists(f), f + ' was created')
+                self.assertFalse(is_repo_file(f), f + ' is repo file')
+                gpg_signed = [
+                    'repo/entry.json',
+                    'repo/index-v1.json',
+                    'repo/index-v2.json',
+                ]
+                self.assertEqual(
+                    (f in gpg_signed or is_repo_file(f, for_gpg_signing=False)),
+                    is_repo_file(f, for_gpg_signing=True),
+                    f + ' gpg signable?',
+                )
+
+            for d in repo_dirs:
+                self.assertTrue(os.path.exists(d), d + ' was created')
+                self.assertFalse(is_repo_file(d), d + ' not repo file')
+
+            for f in non_repo_files:
+                self.assertTrue(os.path.exists(f), f + ' was created')
+                self.assertFalse(is_repo_file(f), f + ' not repo file')
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/gpgsign.TestCase

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import inspect
+import json
+import logging
+import optparse
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
+)
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
+from fdroidserver import common, gpgsign
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+
+class GpgsignTest(unittest.TestCase):
+
+    basedir = Path(__file__).resolve().parent
+
+    def setUp(self):
+        logging.basicConfig(level=logging.DEBUG)
+        self.tempdir = tempfile.TemporaryDirectory()
+        os.chdir(self.tempdir.name)
+        self.repodir = Path('repo')
+        self.repodir.mkdir()
+
+        gpgsign.config = None
+        config = common.read_config(common.options)
+        config['verbose'] = True
+        config['gpghome'] = str((self.basedir / 'gnupghome').resolve())
+        config['gpgkey'] = '1DBA2E89'
+        gpgsign.config = config
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+    @patch('sys.argv', ['fdroid gpgsign', '--verbose'])
+    @patch('fdroidserver.gpgsign.FDroidPopen')
+    def test_sign_index(self, FDroidPopen):
+        """This skips running gpg because its hard to setup in a test env"""
+        index_v1_json = 'repo/index-v1.json'
+        shutil.copy(str(self.basedir / index_v1_json), 'repo')
+        shutil.copy(str(self.basedir / 'SpeedoMeterApp.main_1.apk'), 'repo')
+
+        def _side_effect(gpg):
+            f = gpg[-1]
+            sig = gpg[3]
+            self.assertTrue(sig.startswith(f))
+            open(sig, 'w').close()
+            p = MagicMock()
+            p.returncode = 0
+            return p
+
+        FDroidPopen.side_effect = _side_effect
+        gpgsign.main()
+        self.assertTrue(FDroidPopen.called)
+        self.assertTrue((self.repodir / 'index-v1.json').exists())
+        self.assertTrue((self.repodir / 'index-v1.json.asc').exists())
+        self.assertTrue((self.repodir / 'SpeedoMeterApp.main_1.apk.asc').exists())
+        self.assertFalse((self.repodir / 'index.jar.asc').exists())
+        # smoke check status JSON
+        with (self.repodir / 'status/gpgsign.json').open() as fp:
+            data = json.load(fp)
+        self.assertTrue('index-v1.json' in data['signed'])
+
+
+if __name__ == "__main__":
+    os.chdir(os.path.dirname(__file__))
+
+    parser = optparse.OptionParser()
+    parser.add_option(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Spew out even more information than normal",
+    )
+    (common.options, args) = parser.parse_args(['--verbose'])
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(GpgsignTest))
+    unittest.main(failfast=False)

tests/index.TestCase

@@ -57,7 +57,7 @@ class IndexTest(unittest.TestCase):
         fdroidserver.signindex.config = config
 
         if not os.path.exists('repo/index-v1.jar'):
-            fdroidserver.signindex.sign_index_v1(
+            fdroidserver.signindex.sign_index(
                 os.path.join(self.basedir, 'repo'), 'index-v1.json'
             )
 

tests/signindex.TestCase

@@ -56,17 +56,17 @@ class SignindexTest(unittest.TestCase):
     def tearDown(self):
         self.tempdir.cleanup()
 
-    def test_sign_index_v1(self):
+    def test_sign_index(self):
         shutil.copy(str(self.basedir / 'repo/index-v1.json'), 'repo')
-        signindex.sign_index_v1(str(self.repodir), 'index-v1.json')
+        signindex.sign_index(str(self.repodir), 'index-v1.json')
         self.assertTrue((self.repodir / 'index-v1.jar').exists())
         self.assertTrue((self.repodir / 'index-v1.json').exists())
 
-    def test_sign_index_v1_corrupt(self):
+    def test_sign_index_corrupt(self):
         with open('repo/index-v1.json', 'w') as fp:
             fp.write('corrupt JSON!')
         with self.assertRaises(json.decoder.JSONDecodeError, msg='error on bad JSON'):
-            signindex.sign_index_v1(str(self.repodir), 'index-v1.json')
+            signindex.sign_index(str(self.repodir), 'index-v1.json')
 
     def test_signindex(self):
         os.mkdir('archive')