From d70e5c2cd92eb1924caf51a1f88202749956038f Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 23 May 2022 10:39:17 +0000 Subject: [PATCH] Index v2 --- .gitlab-ci.yml | 1 + fdroidserver/common.py | 41 ++-- fdroidserver/gpgsign.py | 4 +- fdroidserver/index.py | 398 +++++++++++++++++++++++++++++++++++++- fdroidserver/signindex.py | 73 ++++--- fdroidserver/update.py | 46 ++++- tests/common.TestCase | 66 +++++++ tests/gpgsign.TestCase | 91 +++++++++ tests/index.TestCase | 2 +- tests/signindex.TestCase | 8 +- 10 files changed, 677 insertions(+), 53 deletions(-) create mode 100755 tests/gpgsign.TestCase diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3eeff9a6..156c89a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ variables: ci-images-base run-tests: image: registry.gitlab.com/fdroid/ci-images-base script: + - $pip install --upgrade pip setuptools wheel # make this go away: "error: invalid command 'bdist_wheel'" - $pip install -e . - ./tests/run-tests # make sure that translations do not cause stacktraces diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5318d6c1..8cb96cac 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4088,26 +4088,33 @@ def get_per_app_repos(): return repos -def is_repo_file(filename): +def is_repo_file(filename, for_gpg_signing=False): """Whether the file in a repo is a build product to be delivered to users.""" if isinstance(filename, str): filename = filename.encode('utf-8', errors="surrogateescape") - return os.path.isfile(filename) \ - and not filename.endswith(b'.asc') \ - and not filename.endswith(b'.sig') \ - and not filename.endswith(b'.idsig') \ - and not filename.endswith(b'.log.gz') \ - and os.path.basename(filename) not in [ - b'index.css', - b'index.jar', - b'index_unsigned.jar', - b'index.xml', - b'index.html', - b'index.png', - b'index-v1.jar', - b'index-v1.json', - b'categories.txt', - ] + ignore_files = [ + b'categories.txt', + b'entry.jar', + b'index-v1.jar', + b'index-v2.jar', + b'index.css', + b'index.html', + b'index.jar', + b'index.png', + b'index.xml', + b'index_unsigned.jar', + ] + if not for_gpg_signing: + ignore_files += [b'entry.json', b'index-v1.json', b'index-v2.json'] + + return ( + os.path.isfile(filename) + and not filename.endswith(b'.asc') + and not filename.endswith(b'.sig') + and not filename.endswith(b'.idsig') + and not filename.endswith(b'.log.gz') + and os.path.basename(filename) not in ignore_files + ) def get_examples_dir(): diff --git a/fdroidserver/gpgsign.py b/fdroidserver/gpgsign.py index 726805b6..6c8842bd 100644 --- a/fdroidserver/gpgsign.py +++ b/fdroidserver/gpgsign.py @@ -65,9 +65,7 @@ def main(): # Process any apks that are waiting to be signed... for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))): - if common.get_file_extension(f) == 'asc': - continue - if not common.is_repo_file(f) and not f.endswith('/index-v1.json'): + if not common.is_repo_file(f, for_gpg_signing=True): continue filename = os.path.basename(f) sigfilename = filename + ".asc" diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 5ba728e7..fbc6a351 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -28,11 +28,13 @@ import re import shutil import tempfile import urllib.parse +import yaml import zipfile import calendar import qrcode from binascii import hexlify, unhexlify from datetime import datetime, timezone +from pathlib import Path from xml.dom.minidom import Document from . import _ @@ -136,6 +138,8 @@ def make(apps, apks, repodir, archive): fdroid_signing_key_fingerprints) make_v1(sortedapps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints) + make_v2(sortedapps, apks, repodir, repodict, requestsdict, + fdroid_signing_key_fingerprints, archive) make_website(sortedapps, repodir, repodict) @@ -469,6 +473,393 @@ fieldset select, fieldset input, #reposelect select, #reposelect input { }""") +def dict_diff(source, target): + if not isinstance(target, dict) or not isinstance(source, dict): + return target + + result = {key: None for key in source if key not in target} + + for key, value in target.items(): + if key not in source: + result[key] = value + elif value != source[key]: + result[key] = dict_diff(source[key], value) + + return result + + +def file_entry(filename, hashType=None, hsh=None, size=None): + meta = {} + meta["name"] = "/" + filename.split("/", 1)[1] + if hsh: + meta[hashType] = hsh + if hsh != "sha256": + meta["sha256"] = common.sha256sum(filename) + if size: + meta["size"] = size + else: + meta["size"] = os.stat(filename).st_size + return meta + + +def load_locale(name, repodir): + lst = {} + for yml in Path().glob("config/**/{name}.yml".format(name=name)): + locale = yml.parts[1] + if len(yml.parts) == 2: + locale = "en-US" + with open(yml, encoding="utf-8") as fp: + elem = yaml.safe_load(fp) + for akey, avalue in elem.items(): + if akey not in lst: + lst[akey] = {} + for key, value in avalue.items(): + if key not in lst[akey]: + lst[akey][key] = {} + if key == "icon": + shutil.copy(os.path.join("config", value), os.path.join(repodir, "icons")) + lst[akey][key][locale] = file_entry(os.path.join(repodir, "icons", value)) + else: + lst[akey][key][locale] = value + + return lst + + +def convert_datetime(obj): + if isinstance(obj, datetime): + # Java prefers milliseconds + # we also need to account for time zone/daylight saving time + return int(calendar.timegm(obj.timetuple()) * 1000) + return obj + + +def package_metadata(app, repodir): + meta = {} + for element in ( + "added", + # "binaries", + "Categories", + "Changelog", + "IssueTracker", + "lastUpdated", + "License", + "SourceCode", + "Translation", + "WebSite", + "video", + "featureGraphic", + "promoGraphic", + "tvBanner", + "screenshots", + "AuthorEmail", + "AuthorName", + "AuthorPhone", + "AuthorWebSite", + "Bitcoin", + "FlattrID", + "Liberapay", + "LiberapayID", + "Litecoin", + "OpenCollective", + ): + if element in app and app[element]: + element_new = element[:1].lower() + element[1:] + meta[element_new] = convert_datetime(app[element]) + + for element in ( + "Name", + "Summary", + "Description", + ): + element_new = element[:1].lower() + element[1:] + if element in app and app[element]: + meta[element_new] = {"en-US": convert_datetime(app[element])} + elif "localized" in app: + localized = {k: v[element_new] for k, v in app["localized"].items() if element_new in v} + if localized: + meta[element_new] = localized + + if "name" not in meta and app["AutoName"]: + meta["name"] = {"en-US": app["AutoName"]} + + # fdroidserver/metadata.py App default + if meta["license"] == "Unknown": + del meta["license"] + + if app["Donate"]: + meta["donate"] = [app["Donate"]] + + # TODO handle different resolutions + if app.get("icon"): + meta["icon"] = {"en-US": file_entry(os.path.join(repodir, "icons", app["icon"]))} + + if "iconv2" in app: + meta["icon"] = app["iconv2"] + + return meta + + +def convert_version(version, app, repodir): + ver = {} + if "added" in version: + ver["added"] = convert_datetime(version["added"]) + else: + ver["added"] = 0 + + ver["file"] = { + "name": "/{}".format(version["apkName"]), + version["hashType"]: version["hash"], + "size": version["size"] + } + + if "srcname" in version: + ver["src"] = file_entry(os.path.join(repodir, version["srcname"])) + + if "obbMainFile" in version: + ver["obbMainFile"] = file_entry( + os.path.join(repodir, version["obbMainFile"]), + "sha256", version["obbMainFileSha256"] + ) + + if "obbPatchFile" in version: + ver["obbPatchFile"] = file_entry( + os.path.join(repodir, version["obbPatchFile"]), + "sha256", version["obbPatchFileSha256"] + ) + + ver["manifest"] = manifest = {} + + for element in ( + "nativecode", + "versionName", + "maxSdkVersion", + ): + if element in version: + manifest[element] = version[element] + + if "versionCode" in version: + manifest["versionCode"] = int(version["versionCode"]) + + if "features" in version and version["features"]: + manifest["features"] = features = [] + for feature in version["features"]: + # TODO get version from manifest, default (0) is omitted + # features.append({"name": feature, "version": 1}) + features.append({"name": feature}) + + if "minSdkVersion" in version: + manifest["usesSdk"] = {} + manifest["usesSdk"]["minSdkVersion"] = version["minSdkVersion"] + if "targetSdkVersion" in version: + manifest["usesSdk"]["targetSdkVersion"] = version["targetSdkVersion"] + else: + # https://developer.android.com/guide/topics/manifest/uses-sdk-element.html#target + manifest["usesSdk"]["targetSdkVersion"] = manifest["usesSdk"]["minSdkVersion"] + + if "signer" in version: + manifest["signer"] = {"sha256": [version["signer"]]} + + for element in ("uses-permission", "uses-permission-sdk-23"): + en = element.replace("uses-permission", "usesPermission").replace("-sdk-23", "Sdk23") + if element in version and version[element]: + manifest[en] = [] + for perm in version[element]: + if perm[1]: + manifest[en].append({"name": perm[0], "maxSdkVersion": perm[1]}) + else: + manifest[en].append({"name": perm[0]}) + + if "AntiFeatures" in app and app["AntiFeatures"]: + ver["antiFeatures"] = {} + for antif in app["AntiFeatures"]: + # TODO: get reasons from fdroiddata + # ver["antiFeatures"][antif] = {"en-US": "reason"} + ver["antiFeatures"][antif] = {} + + if "AntiFeatures" in version and version["AntiFeatures"]: + if "antiFeatures" not in ver: + ver["antiFeatures"] = {} + for antif in version["AntiFeatures"]: + # TODO: get reasons from fdroiddata + # ver["antiFeatures"][antif] = {"en-US": "reason"} + ver["antiFeatures"][antif] = {} + + if "versionCode" in version: + if int(version["versionCode"]) > int(app["CurrentVersionCode"]): + ver["releaseChannels"] = ["Beta"] + + versionCodeStr = str(version['versionCode']) # TODO build.versionCode should be int! + for build in app.get('Builds', []): + if build['versionCode'] == versionCodeStr and "whatsNew" in build: + ver["whatsNew"] = build["whatsNew"] + break + + return ver + + +def v2_repo(repodict, repodir, archive): + repo = {} + + repo["name"] = {"en-US": repodict["name"]} + repo["description"] = {"en-US": repodict["description"]} + repo["icon"] = {"en-US": file_entry("{}/icons/{}".format(repodir, repodict["icon"]))} + + config = load_locale("config", repodir) + if config: + repo["name"] = config["archive" if archive else "repo"]["name"] + repo["description"] = config["archive" if archive else "repo"]["description"] + repo["icon"] = config["archive" if archive else "repo"]["icon"] + + repo["address"] = repodict["address"] + repo["webBaseUrl"] = "https://f-droid.org/packages/" + + if "repo_url" in common.config: + primary_mirror = common.config["repo_url"][:-len("/repo")] + if "mirrors" in repodict and primary_mirror not in repodict["mirrors"]: + repodict["mirrors"].append(primary_mirror) + + if "mirrors" in repodict: + repo["mirrors"] = [{"url": mirror} for mirror in repodict["mirrors"]] + + repo["timestamp"] = repodict["timestamp"] + + anti_features = load_locale("antiFeatures", repodir) + if anti_features: + repo["antiFeatures"] = anti_features + + categories = load_locale("categories", repodir) + if categories: + repo["categories"] = categories + + channels = load_locale("channels", repodir) + if channels: + repo["releaseChannels"] = channels + + return repo + + +def make_v2(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints, archive): + + def _index_encoder_default(obj): + if isinstance(obj, set): + return sorted(list(obj)) + if isinstance(obj, datetime): + # Java prefers milliseconds + # we also need to account for time zone/daylight saving time + return int(calendar.timegm(obj.timetuple()) * 1000) + if isinstance(obj, dict): + d = collections.OrderedDict() + for key in sorted(obj.keys()): + d[key] = obj[key] + return d + raise TypeError(repr(obj) + " is not JSON serializable") + + output = collections.OrderedDict() + output["repo"] = v2_repo(repodict, repodir, archive) + if requestsdict and requestsdict["install"] or requestsdict["uninstall"]: + output["repo"]["requests"] = requestsdict + + # establish sort order of the index + v1_sort_packages(packages, fdroid_signing_key_fingerprints) + + output_packages = collections.OrderedDict() + output['packages'] = output_packages + for package in packages: + packageName = package['packageName'] + if packageName not in apps: + logging.info(_('Ignoring package without metadata: ') + package['apkName']) + continue + if not package.get('versionName'): + app = apps[packageName] + versionCodeStr = str(package['versionCode']) # TODO build.versionCode should be int! + for build in app.get('Builds', []): + if build['versionCode'] == versionCodeStr: + versionName = build.get('versionName') + logging.info(_('Overriding blank versionName in {apkfilename} from metadata: {version}') + .format(apkfilename=package['apkName'], version=versionName)) + package['versionName'] = versionName + break + if packageName in output_packages: + packagelist = output_packages[packageName] + else: + packagelist = {} + output_packages[packageName] = packagelist + packagelist["metadata"] = package_metadata(apps[packageName], repodir) + if "signer" in package: + packagelist["metadata"]["preferredSigner"] = package["signer"] + + packagelist["versions"] = {} + + packagelist["versions"][package["hash"]] = convert_version(package, apps[packageName], repodir) + + entry = {} + entry["timestamp"] = repodict["timestamp"] + + entry["version"] = repodict["version"] + if "maxage" in repodict: + entry["maxAge"] = repodict["maxage"] + + json_name = 'index-v2.json' + index_file = os.path.join(repodir, json_name) + with open(index_file, "w", encoding="utf-8") as fp: + if common.options.pretty: + json.dump(output, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) + else: + json.dump(output, fp, default=_index_encoder_default, ensure_ascii=False) + + json_name = "tmp/{}_{}.json".format(repodir, convert_datetime(repodict["timestamp"])) + with open(json_name, "w", encoding="utf-8") as fp: + if common.options.pretty: + json.dump(output, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) + else: + json.dump(output, fp, default=_index_encoder_default, ensure_ascii=False) + + entry["index"] = file_entry(index_file) + entry["index"]["numPackages"] = len(output.get("packages", [])) + + indexes = sorted(Path().glob("tmp/{}*.json".format(repodir)), key=lambda x: x.name) + indexes.pop() # remove current index + # remove older indexes + while len(indexes) > 10: + indexes.pop(0).unlink() + + indexes = [json.loads(Path(fn).read_text(encoding="utf-8")) for fn in indexes] + + for diff in Path().glob("{}/diff/*.json".format(repodir)): + diff.unlink() + + entry["diffs"] = {} + for old in indexes: + diff_name = str(old["repo"]["timestamp"]) + ".json" + diff_file = os.path.join(repodir, "diff", diff_name) + diff = dict_diff(old, output) + if not os.path.exists(os.path.join(repodir, "diff")): + os.makedirs(os.path.join(repodir, "diff")) + with open(diff_file, "w", encoding="utf-8") as fp: + if common.options.pretty: + json.dump(diff, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) + else: + json.dump(diff, fp, default=_index_encoder_default, ensure_ascii=False) + + entry["diffs"][old["repo"]["timestamp"]] = file_entry(diff_file) + entry["diffs"][old["repo"]["timestamp"]]["numPackages"] = len(diff.get("packages", [])) + + json_name = "entry.json" + index_file = os.path.join(repodir, json_name) + with open(index_file, "w", encoding="utf-8") as fp: + if common.options.pretty: + json.dump(entry, fp, default=_index_encoder_default, indent=2, ensure_ascii=False) + else: + json.dump(entry, fp, default=_index_encoder_default, ensure_ascii=False) + + if common.options.nosign: + _copy_to_local_copy_dir(repodir, index_file) + logging.debug(_('index-v2 must have a signature, use `fdroid signindex` to create it!')) + else: + signindex.config = common.config + signindex.sign_index(repodir, json_name, signindex.HashAlg.SHA256) + + def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints): def _index_encoder_default(obj): @@ -504,7 +895,10 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ 'ArchivePolicy', 'AutoName', 'AutoUpdateMode', 'MaintainerNotes', 'Provides', 'Repo', 'RepoType', 'RequiresRoot', 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode', - 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'): + 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation', + 'summary', 'description', 'promoGraphic', 'screenshots', 'whatsNew', + 'featureGraphic', 'iconv2', 'tvBanner', + ): continue # name things after the App class fields in fdroidclient @@ -573,7 +967,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!')) else: signindex.config = common.config - signindex.sign_index_v1(repodir, json_name) + signindex.sign_index(repodir, json_name) def _copy_to_local_copy_dir(repodir, f): diff --git a/fdroidserver/signindex.py b/fdroidserver/signindex.py index fae079de..3e532f17 100644 --- a/fdroidserver/signindex.py +++ b/fdroidserver/signindex.py @@ -21,6 +21,7 @@ import os import time import zipfile from argparse import ArgumentParser +from enum import Enum import logging from . import _ @@ -33,29 +34,46 @@ options = None start_timestamp = time.gmtime() -def sign_jar(jar): +HashAlg = Enum("SHA1", "SHA256") + + +def sign_jar(jar, hash_algorithm=None): """Sign a JAR file with Java's jarsigner. This method requires a properly initialized config object. - - This does use old hashing algorithms, i.e. SHA1, but that's not - broken yet for file verification. This could be set to SHA256, - but then Android < 4.3 would not be able to verify it. - https://code.google.com/p/android/issues/detail?id=38321 """ - args = [ - config['jarsigner'], - '-keystore', - config['keystore'], - '-storepass:env', - 'FDROID_KEY_STORE_PASS', - '-digestalg', - 'SHA1', - '-sigalg', - 'SHA1withRSA', - jar, - config['repo_keyalias'], - ] + if hash_algorithm == HashAlg.SHA256: + args = [ + config['jarsigner'], + '-keystore', + config['keystore'], + '-storepass:env', + 'FDROID_KEY_STORE_PASS', + '-digestalg', + 'SHA-256', + '-sigalg', + 'SHA256withRSA', + jar, + config['repo_keyalias'], + ] + else: + # This does use old hashing algorithms, i.e. SHA1, but that's not + # broken yet for file verification. This could be set to SHA256, + # but then Android < 4.3 would not be able to verify it. + # https://code.google.com/p/android/issues/detail?id=38321 + args = [ + config['jarsigner'], + '-keystore', + config['keystore'], + '-storepass:env', + 'FDROID_KEY_STORE_PASS', + '-digestalg', + 'SHA1', + '-sigalg', + 'SHA1withRSA', + jar, + config['repo_keyalias'], + ] if config['keystore'] == 'NONE': args += config['smartcardoptions'] else: # smardcards never use -keypass @@ -69,7 +87,7 @@ def sign_jar(jar): raise FDroidException("Failed to sign %s!" % jar) -def sign_index_v1(repodir, json_name): +def sign_index(repodir, json_name, hash_algorithm=None): """Sign index-v1.json to make index-v1.jar. This is a bit different than index.jar: instead of their being index.xml @@ -84,12 +102,14 @@ def sign_index_v1(repodir, json_name): # Test if index is valid with open(index_file, encoding="utf-8") as fp: index = json.load(fp) - [metadata.App(app) for app in index["apps"]] + # TODO: add test for index-v2 + if "apps" in index: + [metadata.App(app) for app in index["apps"]] jar_file = os.path.join(repodir, name + '.jar') with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar: jar.write(index_file, json_name) - sign_jar(jar_file) + sign_jar(jar_file, hash_algorithm) def status_update_json(signed): @@ -138,7 +158,14 @@ def main(): json_name = 'index-v1.json' index_file = os.path.join(output_dir, json_name) if os.path.exists(index_file): - sign_index_v1(output_dir, json_name) + sign_index(output_dir, json_name) + logging.info('Signed ' + index_file) + signed.append(index_file) + + json_name = 'entry.json' + index_file = os.path.join(output_dir, json_name) + if os.path.exists(index_file): + sign_index(output_dir, json_name, HashAlg.SHA256) logging.info('Signed ' + index_file) signed.append(index_file) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 4db8a2cc..7d88bdde 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -554,7 +554,7 @@ def _get_localized_dict(app, locale): return app['localized'][locale] -def _set_localized_text_entry(app, locale, key, f): +def _set_localized_text_entry(app, locale, key, f, versionCode=None): """Read a fastlane/triple-t metadata file and add an entry to the app. This reads more than the limit, in case there is leading or @@ -563,9 +563,17 @@ def _set_localized_text_entry(app, locale, key, f): """ try: limit = config['char_limits'][key] - localized = _get_localized_dict(app, locale) + if not versionCode: + localized = _get_localized_dict(app, locale) with open(f, errors='replace') as fp: text = fp.read(limit * 2) + if versionCode: + for build in app["Builds"]: + if int(build["versionCode"]) == versionCode: + if "whatsNew" not in build: + build["whatsNew"] = collections.OrderedDict() + build["whatsNew"][locale] = text[:limit] + return if len(text) > 0: if key in ('name', 'summary', 'video'): # hardcoded as a single line localized[key] = text.strip('\n')[:limit] @@ -987,15 +995,33 @@ def insert_localized_app_metadata(apps): locale = segments[-2] _set_localized_text_entry(apps[packageName], locale, 'whatsNew', os.path.join(root, f)) - continue base, extension = common.get_extension(f) + + if extension == 'txt': + try: + versionCode = int(base) + locale = segments[-2] + if base in [a["versionCode"] for a in apps[packageName]["Builds"]]: + _set_localized_text_entry(apps[packageName], locale, 'whatsNew', + os.path.join(root, f), versionCode) + continue + except ValueError: + pass + if locale == 'images': locale = segments[-2] destdir = os.path.join('repo', packageName, locale) if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS: os.makedirs(destdir, mode=0o755, exist_ok=True) _strip_and_copy_image(os.path.join(root, f), destdir) + dst = os.path.join(destdir, f) + if os.path.isfile(dst): + if base == "icon": + base = "iconv2" + if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict): + apps[packageName][base] = collections.OrderedDict() + apps[packageName][base][locale] = index.file_entry(dst) for d in dirs: if d in SCREENSHOT_DIRS: if locale == 'images': @@ -1044,12 +1070,26 @@ def insert_localized_app_metadata(apps): if not os.path.exists(index_file): os.link(f, index_file, follow_symlinks=False) graphics[base] = filename + if base == "icon": + base = "iconv2" + if base not in apps[packageName] or not isinstance(apps[packageName][base], collections.OrderedDict): + apps[packageName][base] = collections.OrderedDict() + apps[packageName][base][locale] = index.file_entry(index_file) elif screenshotdir in SCREENSHOT_DIRS: # there can any number of these per locale logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f)) if screenshotdir not in graphics: graphics[screenshotdir] = [] graphics[screenshotdir].append(filename) + + newKey = screenshotdir.replace("Screenshots", "") + if "screenshots" not in apps[packageName]: + apps[packageName]["screenshots"] = collections.OrderedDict() + if newKey not in apps[packageName]["screenshots"]: + apps[packageName]["screenshots"][newKey] = collections.OrderedDict() + if locale not in apps[packageName]["screenshots"][newKey]: + apps[packageName]["screenshots"][newKey][locale] = [] + apps[packageName]["screenshots"][newKey][locale].append(index.file_entry(f)) else: logging.warning(_('Unsupported graphics file found: {path}').format(path=f)) diff --git a/tests/common.TestCase b/tests/common.TestCase index a29920be..e77d7b34 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -2372,6 +2372,72 @@ class CommonTest(unittest.TestCase): fdroidserver.common.set_FDroidPopen_env(build) self.assertNotIn('', os.getenv('PATH').split(os.pathsep)) + def test_is_repo_file(self): + is_repo_file = fdroidserver.common.is_repo_file + self.assertFalse(is_repo_file('does-not-exist')) + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + Path('repo').mkdir() + repo_files = [ + 'repo/com.example.test.helloworld_1.apk', + 'repo/com.politedroid_6.apk', + 'repo/duplicate.permisssions_9999999.apk', + 'repo/fake.ota.update_1234.zip', + 'repo/info.guardianproject.index-v1.jar_123.apk', + 'repo/info.zwanenburg.caffeinetile_4.apk', + 'repo/main.1101613.obb.main.twoversions.obb', + ] + index_files = [ + 'repo/entry.jar', + 'repo/entry.json', + 'repo/index-v1.jar', + 'repo/index-v1.json', + 'repo/index-v2.jar', + 'repo/index-v2.json', + 'repo/index.css', + 'repo/index.html', + 'repo/index.jar', + 'repo/index.png', + 'repo/index.xml', + ] + non_repo_files = ['repo/categories.txt'] + for f in repo_files + index_files + non_repo_files: + open(f, 'w').close() + + repo_dirs = [ + 'repo/com.politedroid', + 'repo/info.guardianproject.index-v1.jar', + 'repo/status', + ] + for d in repo_dirs: + os.mkdir(d) + + for f in repo_files: + self.assertTrue(os.path.exists(f), f + ' was created') + self.assertTrue(is_repo_file(f), f + ' is repo file') + + for f in index_files: + self.assertTrue(os.path.exists(f), f + ' was created') + self.assertFalse(is_repo_file(f), f + ' is repo file') + gpg_signed = [ + 'repo/entry.json', + 'repo/index-v1.json', + 'repo/index-v2.json', + ] + self.assertEqual( + (f in gpg_signed or is_repo_file(f, for_gpg_signing=False)), + is_repo_file(f, for_gpg_signing=True), + f + ' gpg signable?', + ) + + for d in repo_dirs: + self.assertTrue(os.path.exists(d), d + ' was created') + self.assertFalse(is_repo_file(d), d + ' not repo file') + + for f in non_repo_files: + self.assertTrue(os.path.exists(f), f + ' was created') + self.assertFalse(is_repo_file(f), f + ' not repo file') + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/gpgsign.TestCase b/tests/gpgsign.TestCase new file mode 100755 index 00000000..a4e8a0d3 --- /dev/null +++ b/tests/gpgsign.TestCase @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import inspect +import json +import logging +import optparse +import os +import shutil +import sys +import tempfile +import unittest + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') +) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +from fdroidserver import common, gpgsign +from pathlib import Path +from unittest.mock import MagicMock, patch + + +class GpgsignTest(unittest.TestCase): + + basedir = Path(__file__).resolve().parent + + def setUp(self): + logging.basicConfig(level=logging.DEBUG) + self.tempdir = tempfile.TemporaryDirectory() + os.chdir(self.tempdir.name) + self.repodir = Path('repo') + self.repodir.mkdir() + + gpgsign.config = None + config = common.read_config(common.options) + config['verbose'] = True + config['gpghome'] = str((self.basedir / 'gnupghome').resolve()) + config['gpgkey'] = '1DBA2E89' + gpgsign.config = config + + def tearDown(self): + self.tempdir.cleanup() + + @patch('sys.argv', ['fdroid gpgsign', '--verbose']) + @patch('fdroidserver.gpgsign.FDroidPopen') + def test_sign_index(self, FDroidPopen): + """This skips running gpg because its hard to setup in a test env""" + index_v1_json = 'repo/index-v1.json' + shutil.copy(str(self.basedir / index_v1_json), 'repo') + shutil.copy(str(self.basedir / 'SpeedoMeterApp.main_1.apk'), 'repo') + + def _side_effect(gpg): + f = gpg[-1] + sig = gpg[3] + self.assertTrue(sig.startswith(f)) + open(sig, 'w').close() + p = MagicMock() + p.returncode = 0 + return p + + FDroidPopen.side_effect = _side_effect + gpgsign.main() + self.assertTrue(FDroidPopen.called) + self.assertTrue((self.repodir / 'index-v1.json').exists()) + self.assertTrue((self.repodir / 'index-v1.json.asc').exists()) + self.assertTrue((self.repodir / 'SpeedoMeterApp.main_1.apk.asc').exists()) + self.assertFalse((self.repodir / 'index.jar.asc').exists()) + # smoke check status JSON + with (self.repodir / 'status/gpgsign.json').open() as fp: + data = json.load(fp) + self.assertTrue('index-v1.json' in data['signed']) + + +if __name__ == "__main__": + os.chdir(os.path.dirname(__file__)) + + parser = optparse.OptionParser() + parser.add_option( + "-v", + "--verbose", + action="store_true", + default=False, + help="Spew out even more information than normal", + ) + (common.options, args) = parser.parse_args(['--verbose']) + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(GpgsignTest)) + unittest.main(failfast=False) diff --git a/tests/index.TestCase b/tests/index.TestCase index 0c8b80ab..b101eb46 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -57,7 +57,7 @@ class IndexTest(unittest.TestCase): fdroidserver.signindex.config = config if not os.path.exists('repo/index-v1.jar'): - fdroidserver.signindex.sign_index_v1( + fdroidserver.signindex.sign_index( os.path.join(self.basedir, 'repo'), 'index-v1.json' ) diff --git a/tests/signindex.TestCase b/tests/signindex.TestCase index 7966821d..98ff8313 100755 --- a/tests/signindex.TestCase +++ b/tests/signindex.TestCase @@ -56,17 +56,17 @@ class SignindexTest(unittest.TestCase): def tearDown(self): self.tempdir.cleanup() - def test_sign_index_v1(self): + def test_sign_index(self): shutil.copy(str(self.basedir / 'repo/index-v1.json'), 'repo') - signindex.sign_index_v1(str(self.repodir), 'index-v1.json') + signindex.sign_index(str(self.repodir), 'index-v1.json') self.assertTrue((self.repodir / 'index-v1.jar').exists()) self.assertTrue((self.repodir / 'index-v1.json').exists()) - def test_sign_index_v1_corrupt(self): + def test_sign_index_corrupt(self): with open('repo/index-v1.json', 'w') as fp: fp.write('corrupt JSON!') with self.assertRaises(json.decoder.JSONDecodeError, msg='error on bad JSON'): - signindex.sign_index_v1(str(self.repodir), 'index-v1.json') + signindex.sign_index(str(self.repodir), 'index-v1.json') def test_signindex(self): os.mkdir('archive')