From c9aa26d89ea077707df7b8b3d4a4e1b3e6c37ca6 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 28 Nov 2016 21:09:07 +0100 Subject: [PATCH] add index V1 format, a direct translation of internal dict Python encode/decode libs work directly with dicts, so the internal dict can just be passed directly to any of these libs (pyyaml, pyjson, msgpack, simplejson, etc). This still generates the exact same index.xml as before. This converts the internal format for the repo timestamp to a datetime instance, which can be easily converted to UNIX time in seconds for XML and UNIX time in milliseconds for the new index formats. UNIX time in milliseconds is directly serialized into a java.util.Date instance by Jackson. --- .gitignore | 2 + fdroidserver/common.py | 2 + fdroidserver/server.py | 2 + fdroidserver/update.py | 259 +++++++++++++++++++++++++++++------------ tests/run-tests | 8 ++ 5 files changed, 196 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 4652e8a2..69c27f93 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ makebuildserver.config.py /tests/archive/icons* /tests/archive/index.jar /tests/archive/index.xml +/tests/archive/index-v1.jar /tests/repo/index.jar +/tests/repo/index-v1.jar /tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk /unsigned/ diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 673d57af..c84ddfff 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2192,5 +2192,7 @@ def is_repo_file(filename): 'index_unsigned.jar', 'index.xml', 'index.html', + 'index-v1.jar', + 'index-v1.json', 'categories.txt', ] diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 998b80cf..e2969037 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -137,6 +137,7 @@ def update_serverwebroot(serverwebroot, repo_section): rsyncargs += ['-e', 'ssh -i ' + config['identity_file']] indexxml = os.path.join(repo_section, 'index.xml') indexjar = os.path.join(repo_section, 'index.jar') + indexv1jar = os.path.join(repo_section, 'index-v1.jar') # Upload the first time without the index files and delay the deletion as # much as possible, that keeps the repo functional while this update is # running. Then once it is complete, rerun the command again to upload @@ -147,6 +148,7 @@ def update_serverwebroot(serverwebroot, repo_section): logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) if subprocess.call(rsyncargs + ['--exclude', indexxml, '--exclude', indexjar, + '--exclude', indexv1jar, repo_section, serverwebroot]) != 0: sys.exit(1) if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 5356f800..6a22c302 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -19,6 +19,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import copy import sys import os import shutil @@ -1013,6 +1014,171 @@ def make_index(apps, sortedids, apks, repodir, archive): :param categories: list of categories """ + def _resolve_description_link(appid): + if appid in apps: + return ("fdroid.app:" + appid, apps[appid].Name) + raise MetaDataException("Cannot resolve app id " + appid) + + nosigningkey = False + if not options.nosign: + if 'repo_keyalias' not in config: + nosigningkey = True + logging.critical("'repo_keyalias' not found in config.py!") + if 'keystore' not in config: + nosigningkey = True + logging.critical("'keystore' not found in config.py!") + if 'keystorepass' not in config and 'keystorepassfile' not in config: + nosigningkey = True + logging.critical("'keystorepass' not found in config.py!") + if 'keypass' not in config and 'keypassfile' not in config: + nosigningkey = True + logging.critical("'keypass' not found in config.py!") + if not os.path.exists(config['keystore']): + nosigningkey = True + logging.critical("'" + config['keystore'] + "' does not exist!") + if nosigningkey: + logging.warning("`fdroid update` requires a signing key, you can create one using:") + logging.warning("\tfdroid update --create-key") + sys.exit(1) + + repodict = collections.OrderedDict() + repodict['timestamp'] = datetime.utcnow() + repodict['version'] = METADATA_VERSION + + if config['repo_maxage'] != 0: + repodict['maxage'] = config['repo_maxage'] + + if archive: + repodict['name'] = config['archive_name'] + repodict['icon'] = os.path.basename(config['archive_icon']) + repodict['address'] = config['archive_url'] + repodict['description'] = config['archive_description'] + urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path) + else: + repodict['name'] = config['repo_name'] + repodict['icon'] = os.path.basename(config['repo_icon']) + repodict['address'] = config['repo_url'] + repodict['description'] = config['repo_description'] + urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path) + + mirrorcheckfailed = False + mirrors = [] + for mirror in sorted(config.get('mirrors', [])): + base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) + if config.get('nonstandardwebroot') is not True and base != 'fdroid': + logging.error("mirror '" + mirror + "' does not end with 'fdroid'!") + mirrorcheckfailed = True + # must end with / or urljoin strips a whole path segment + if mirror.endswith('/'): + mirrors.append(urllib.parse.urljoin(mirror, urlbasepath)) + else: + mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath)) + for mirror in config.get('servergitmirrors', []): + mirror = get_raw_mirror(mirror) + if mirror is not None: + mirrors.append(mirror + '/') + if mirrorcheckfailed: + sys.exit(1) + if mirrors: + repodict['mirrors'] = mirrors + + appsWithPackages = collections.OrderedDict() + for packageName in sortedids: + app = apps[packageName] + if app['Disabled']: + continue + + # only include apps with packages + for apk in apks: + if apk['packageName'] == packageName: + newapp = copy.copy(app) # update wiki needs unmodified description + newapp['Description'] = metadata.description_html(app['Description'], + _resolve_description_link) + appsWithPackages[packageName] = newapp + break + + make_index_v0(appsWithPackages, apks, repodir, repodict) + make_index_v1(appsWithPackages, apks, repodir, repodict) + + +def make_index_v1(apps, packages, repodir, repodict): + + def _index_encoder_default(obj): + if isinstance(obj, set): + return list(obj) + if isinstance(obj, datetime): + return int(obj.timestamp() * 1000) # Java expects milliseconds + raise TypeError(repr(obj) + " is not JSON serializable") + + output = collections.OrderedDict() + output['repo'] = repodict + + appslist = [] + output['apps'] = appslist + for appid, appdict in apps.items(): + d = collections.OrderedDict() + appslist.append(d) + for k, v in sorted(appdict.items()): + if not v: + continue + if k in ('builds', 'comments', 'metadatapath', + 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes', + 'Provides', 'Repo', 'RepoType', 'RequiresRoot', + 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode', + 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'): + continue + + # name things after the App class fields in fdroidclient + if k == 'id': + k = 'packageName' + elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name + k = 'suggestedVersionCode' + elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name + k = 'suggestedVersionName' + elif k == 'AutoName': + if 'Name' not in apps[appid]: + d['name'] = v + continue + else: + k = k[:1].lower() + k[1:] + d[k] = v + + output_packages = dict() + output['packages'] = output_packages + for package in packages: + packageName = package['packageName'] + if packageName in output_packages: + packagelist = output_packages[packageName] + else: + packagelist = [] + output_packages[packageName] = packagelist + d = collections.OrderedDict() + packagelist.append(d) + for k, v in sorted(package.items()): + if not v: + continue + if k in ('icon', 'icons', 'icons_src', 'name', ): + continue + d[k] = v + + json_name = 'index-v1.json' + index_file = os.path.join(repodir, json_name) + with open(index_file, 'w') as fp: + json.dump(output, fp, default=_index_encoder_default) + + if options.nosign: + logging.debug('index-v1 must have a signature, signindex will overwrite it!') + + jar_file = os.path.join(repodir, 'index-v1.jar') + with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar: + jar.write(index_file, json_name) + signjar(jar_file) + os.remove(index_file) + + +def make_index_v0(apps, apks, repodir, repodict): + '''aka index.jar aka index.xml''' + doc = Document() def addElement(name, value, doc, parent): @@ -1041,71 +1207,17 @@ def make_index(apps, sortedids, apks, repodir, archive): repoel = doc.createElement("repo") - mirrorcheckfailed = False - mirrors = [] - for mirror in sorted(config.get('mirrors', [])): - base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) - if config.get('nonstandardwebroot') is not True and base != 'fdroid': - logging.error("mirror '" + mirror + "' does not end with 'fdroid'!") - mirrorcheckfailed = True - # must end with / or urljoin strips a whole path segment - if mirror.endswith('/'): - mirrors.append(mirror) - else: - mirrors.append(mirror + '/') - for mirror in config.get('servergitmirrors', []): - mirror = get_raw_mirror(mirror) - if mirror is not None: - mirrors.append(mirror + '/') - if mirrorcheckfailed: - sys.exit(1) + repoel.setAttribute("name", repodict['name']) + if 'maxage' in repodict: + repoel.setAttribute("maxage", str(repodict['maxage'])) + repoel.setAttribute("icon", os.path.basename(repodict['icon'])) + repoel.setAttribute("url", repodict['address']) + addElement('description', repodict['description'], doc, repoel) + for mirror in repodict.get('mirrors', []): + addElement('mirror', mirror, doc, repoel) - if archive: - repoel.setAttribute("name", config['archive_name']) - if config['repo_maxage'] != 0: - repoel.setAttribute("maxage", str(config['repo_maxage'])) - repoel.setAttribute("icon", os.path.basename(config['archive_icon'])) - repoel.setAttribute("url", config['archive_url']) - addElement('description', config['archive_description'], doc, repoel) - urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path) - for mirror in mirrors: - addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel) - - else: - repoel.setAttribute("name", config['repo_name']) - if config['repo_maxage'] != 0: - repoel.setAttribute("maxage", str(config['repo_maxage'])) - repoel.setAttribute("icon", os.path.basename(config['repo_icon'])) - repoel.setAttribute("url", config['repo_url']) - addElement('description', config['repo_description'], doc, repoel) - urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path) - for mirror in mirrors: - addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel) - - repoel.setAttribute("version", str(METADATA_VERSION)) - repoel.setAttribute("timestamp", str(int(time.time()))) - - nosigningkey = False - if not options.nosign: - if 'repo_keyalias' not in config: - nosigningkey = True - logging.critical("'repo_keyalias' not found in config.py!") - if 'keystore' not in config: - nosigningkey = True - logging.critical("'keystore' not found in config.py!") - if 'keystorepass' not in config and 'keystorepassfile' not in config: - nosigningkey = True - logging.critical("'keystorepass' not found in config.py!") - if 'keypass' not in config and 'keypassfile' not in config: - nosigningkey = True - logging.critical("'keypass' not found in config.py!") - if not os.path.exists(config['keystore']): - nosigningkey = True - logging.critical("'" + config['keystore'] + "' does not exist!") - if nosigningkey: - logging.warning("`fdroid update` requires a signing key, you can create one using:") - logging.warning("\tfdroid update --create-key") - sys.exit(1) + repoel.setAttribute("version", str(repodict['version'])) + repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp()) repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8')) root.appendChild(repoel) @@ -1125,8 +1237,8 @@ def make_index(apps, sortedids, apks, repodir, archive): root.appendChild(element) element.setAttribute('packageName', packageName) - for appid in sortedids: - app = metadata.App(apps[appid]) + for appid, appdict in apps.items(): + app = metadata.App(appdict) if app.Disabled is not None: continue @@ -1154,18 +1266,11 @@ def make_index(apps, sortedids, apks, repodir, archive): if app.icon: addElement('icon', app.icon, doc, apel) - def linkres(appid): - if appid in apps: - return ("fdroid.app:" + appid, apps[appid].Name) - raise MetaDataException("Cannot resolve app id " + appid) - if app.get('Description'): description = app.Description else: - description = 'No description available' - addElement('desc', - metadata.description_html(description, linkres), - doc, apel) + description = '

No description available

' + addElement('desc', description, doc, apel) addElement('license', app.License, doc, apel) if app.Categories: addElement('categories', ','.join(app.Categories), doc, apel) @@ -1491,11 +1596,11 @@ def make_binary_transparency_log(repodirs): cpdir = os.path.join(btrepo, repodir) if not os.path.exists(cpdir): os.mkdir(cpdir) - for f in ('index.xml', ): + for f in ('index.xml', 'index-v1.json'): dest = os.path.join(cpdir, f) shutil.copyfile(os.path.join(repodir, f), dest) gitrepo.index.add([os.path.join(repodir, f), ]) - for f in ('index.jar', ): + for f in ('index.jar', 'index-v1.jar'): repof = os.path.join(repodir, f) dest = os.path.join(cpdir, f) jarin = zipfile.ZipFile(repof, 'r') diff --git a/tests/run-tests b/tests/run-tests index 66c4b2a1..f48acfff 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -169,6 +169,7 @@ echo "mirrors = ('http://foobarfoobarfoobar.onion/fdroid','https://foo.bar/fdroi $fdroid update --verbose --pretty test -e repo/index.xml test -e repo/index.jar +test -e repo/index-v1.jar grep -F '> config.py $fdroid update --verbose --pretty test -e repo/index.xml test -e repo/index.jar +test -e repo/index-v1.jar grep -F '