Merge branch 'index-v1' into 'master'

app index V1 - support graphics, localization, and more

See merge request !221

.gitignore

@@ -36,6 +36,8 @@ makebuildserver.config.py
 /tests/archive/icons*
 /tests/archive/index.jar
 /tests/archive/index.xml
+/tests/archive/index-v1.jar
 /tests/repo/index.jar
+/tests/repo/index-v1.jar
 /tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk
 /unsigned/

examples/config.py

@@ -84,7 +84,7 @@ The repository of older versions of applications from the main demo repository.
 # By default, the "current version" link will be based on the "Name" of the
 # app from the metadata. You can change it to use a different field from the
 # metadata here:
-# current_version_name_source = 'id'
+# current_version_name_source = 'packageName'
 
 # Optionally, override home directory for gpg
 # gpghome = '/home/fdroid/somewhere/else/.gnupg'
@@ -247,7 +247,8 @@ The repository of older versions of applications from the main demo repository.
 
 # Only set this to true when running a repository where you want to generate
 # stats, and only then on the master build servers, not a development
-# machine.
+# machine. If you want to keep the "added" and "last updated" dates for each
+# app and APK in your repo, then you should enable this.
 # update_stats = True
 
 # When used with stats, this is a list of IP addresses that are ignored for

fdroidserver/build.py

@@ -1005,7 +1005,7 @@ def parse_commandline():
 
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
     parser.add_argument("-l", "--latest", action="store_true", default=False,
                         help="Build only the latest version of each package")
     parser.add_argument("-s", "--stop", action="store_true", default=False,

fdroidserver/common.py

@@ -34,8 +34,10 @@ import logging
 import hashlib
 import socket
 import base64
+import zipfile
 import xml.etree.ElementTree as XMLElementTree
 
+from datetime import datetime
 from distutils.version import LooseVersion
 from queue import Queue
 from zipfile import ZipFile
@@ -386,6 +388,47 @@ def write_password_file(pwtype, password=None):
     config[pwtype + 'file'] = filename
 
 
+def signjar(jar):
+    '''
+    sign a JAR file with Java's jarsigner.
+
+    This does use old hashing algorithms, i.e. SHA1, but that's not
+    broken yet for file verification.  This could be set to SHA256,
+    but then Android < 4.3 would not be able to verify it.
+    https://code.google.com/p/android/issues/detail?id=38321
+    '''
+    args = [config['jarsigner'], '-keystore', config['keystore'],
+            '-storepass:file', config['keystorepassfile'],
+            '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
+            jar, config['repo_keyalias']]
+    if config['keystore'] == 'NONE':
+        args += config['smartcardoptions']
+    else:  # smardcards never use -keypass
+        args += ['-keypass:file', config['keypassfile']]
+    p = FDroidPopen(args)
+    if p.returncode != 0:
+        logging.critical("Failed to sign %s!" % jar)
+        sys.exit(1)
+
+
+def sign_index_v1(repodir, json_name):
+    """
+    sign index-v1.json to make index-v1.jar
+
+    This is a bit different than index.jar: instead of their being index.xml
+    and index_unsigned.jar, the presense of index-v1.json means that there is
+    unsigned data.  That file is then stuck into a jar and signed by the
+    signing process.  index-v1.json is never published to the repo.  It is
+    included in the binary transparency log, if that is enabled.
+    """
+    name, ext = get_extension(json_name)
+    index_file = os.path.join(repodir, json_name)
+    jar_file = os.path.join(repodir, name + '.jar')
+    with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
+        jar.write(index_file, json_name)
+    signjar(jar_file)
+
+
 def get_local_metadata_files():
     '''get any metadata files local to an app's source repo
 
@@ -1624,7 +1667,7 @@ class KnownApks:
                     if len(t) == 2:
                         self.apks[t[0]] = (t[1], None)
                     else:
-                        self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
+                        self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
         self.changed = False
 
     def writeifchanged(self):
@@ -1639,19 +1682,21 @@ class KnownApks:
             appid, added = app
             line = apk + ' ' + appid
             if added:
-                line += ' ' + time.strftime('%Y-%m-%d', added)
+                line += ' ' + added.strftime('%Y-%m-%d')
             lst.append(line)
 
         with open(self.path, 'w', encoding='utf8') as f:
             for line in sorted(lst, key=natural_key):
                 f.write(line + '\n')
 
-    # Record an apk (if it's new, otherwise does nothing)
-    # Returns the date it was added.
     def recordapk(self, apk, app, default_date=None):
+        '''
+        Record an apk (if it's new, otherwise does nothing)
+        Returns the date it was added as a datetime instance
+        '''
         if apk not in self.apks:
             if default_date is None:
-                default_date = time.gmtime(time.time())
+                default_date = datetime.utcnow()
             self.apks[apk] = (app, default_date)
             self.changed = True
         _, added = self.apks[apk]
@@ -2192,5 +2237,7 @@ def is_repo_file(filename):
             'index_unsigned.jar',
             'index.xml',
             'index.html',
+            'index-v1.jar',
+            'index-v1.json',
             'categories.txt',
         ]

fdroidserver/install.py

@@ -48,7 +48,7 @@ def main():
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
     parser.add_argument("-a", "--all", action="store_true", default=False,
                         help="Install all signed applications available")
     options = parser.parse_args()

fdroidserver/metadata.py

@@ -813,6 +813,11 @@ def post_metadata_parse(app):
         if type(v) in (float, int):
             app[k] = str(v)
 
+    if isinstance(app.Categories, str):
+        app.Categories = [app.Categories]
+    else:
+        app.Categories = [str(i) for i in app.Categories]
+
     builds = []
     if 'builds' in app:
         for build in app['builds']:
@@ -831,9 +836,6 @@ def post_metadata_parse(app):
                     build[k] = str(v)
             builds.append(build)
 
-    if not app.get('Description'):
-        app['Description'] = 'No description available'
-
     app.builds = sorted_builds(builds)
 
 

fdroidserver/publish.py

@@ -41,7 +41,7 @@ def main():
     parser = ArgumentParser(usage="%(prog)s [options] "
                             "[APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W

fdroidserver/scanner.py

@@ -251,7 +251,7 @@ def main():
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
     metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
     metadata.warnings_action = options.W

fdroidserver/server.py

@@ -137,6 +137,7 @@ def update_serverwebroot(serverwebroot, repo_section):
         rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
     indexxml = os.path.join(repo_section, 'index.xml')
     indexjar = os.path.join(repo_section, 'index.jar')
+    indexv1jar = os.path.join(repo_section, 'index-v1.jar')
     # Upload the first time without the index files and delay the deletion as
     # much as possible, that keeps the repo functional while this update is
     # running.  Then once it is complete, rerun the command again to upload
@@ -147,6 +148,7 @@ def update_serverwebroot(serverwebroot, repo_section):
     logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
     if subprocess.call(rsyncargs +
                        ['--exclude', indexxml, '--exclude', indexjar,
+                        '--exclude', indexv1jar,
                         repo_section, serverwebroot]) != 0:
         sys.exit(1)
     if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:

fdroidserver/signindex.py

@@ -22,7 +22,6 @@ from argparse import ArgumentParser
 import logging
 
 from . import common
-from .common import FDroidPopen
 
 config = None
 options = None
@@ -55,23 +54,19 @@ def main():
 
         unsigned = os.path.join(output_dir, 'index_unsigned.jar')
         if os.path.exists(unsigned):
-
+            common.signjar(unsigned)
-            args = [config['jarsigner'], '-keystore', config['keystore'],
-                    '-storepass:file', config['keystorepassfile'],
-                    '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
-                    unsigned, config['repo_keyalias']]
-            if config['keystore'] == 'NONE':
-                args += config['smartcardoptions']
-            else:  # smardcards never use -keypass
-                args += ['-keypass:file', config['keypassfile']]
-            p = FDroidPopen(args)
-            if p.returncode != 0:
-                logging.critical("Failed to sign index")
-                sys.exit(1)
             os.rename(unsigned, os.path.join(output_dir, 'index.jar'))
             logging.info('Signed index in ' + output_dir)
             signed += 1
 
+        json_name = 'index-v1.json'
+        index_file = os.path.join(output_dir, json_name)
+        if os.path.exists(index_file):
+            common.sign_index_v1(output_dir, json_name)
+            os.remove(index_file)
+            logging.info('Signed ' + index_file)
+            signed += 1
+
     if signed == 0:
         logging.info("Nothing to do")
 

fdroidserver/update.py

@@ -19,6 +19,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import sys
 import os
 import shutil
@@ -34,7 +35,6 @@ import urllib.parse
 from datetime import datetime, timedelta
 from xml.dom.minidom import Document
 from argparse import ArgumentParser
-import time
 
 import collections
 from pyasn1.error import PyAsn1Error
@@ -116,8 +116,8 @@ def update_wiki(apps, sortedids, apks):
         wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
             appid,
             app.Name,
-            time.strftime('%Y-%m-%d', app.added) if app.added else '',
+            app.added.strftime('%Y-%m-%d') if app.added else '',
-            time.strftime('%Y-%m-%d', app.lastUpdated) if app.lastUpdated else '',
+            app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
             app.SourceCode,
             app.IssueTracker,
             app.WebSite,
@@ -151,8 +151,8 @@ def update_wiki(apps, sortedids, apks):
         cantupdate = False
         buildfails = False
         for apk in apks:
-            if apk['id'] == appid:
+            if apk['packageName'] == appid:
-                if str(apk['versioncode']) == app.CurrentVersionCode:
+                if str(apk['versionCode']) == app.CurrentVersionCode:
                     gotcurrentver = True
                 apklist.append(apk)
         # Include ones we can't build, as a special case...
@@ -161,26 +161,26 @@ def update_wiki(apps, sortedids, apks):
                 if build.versionCode == app.CurrentVersionCode:
                     cantupdate = True
                 # TODO: Nasty: vercode is a string in the build, and an int elsewhere
-                apklist.append({'versioncode': int(build.versionCode),
+                apklist.append({'versionCode': int(build.versionCode),
-                                'version': build.versionName,
+                                'versionName': build.versionName,
                                 'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
                                 })
             else:
                 builtit = False
                 for apk in apklist:
-                    if apk['versioncode'] == int(build.versionCode):
+                    if apk['versionCode'] == int(build.versionCode):
                         builtit = True
                         break
                 if not builtit:
                     buildfails = True
-                    apklist.append({'versioncode': int(build.versionCode),
+                    apklist.append({'versionCode': int(build.versionCode),
-                                    'version': build.versionName,
+                                    'versionName': build.versionName,
                                     'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.versionCode),
                                     })
         if app.CurrentVersionCode == '0':
             cantupdate = True
         # Sort with most recent first...
-        apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
+        apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 
         wikidata += "=Versions=\n"
         if len(apklist) == 0:
@@ -198,7 +198,7 @@ def update_wiki(apps, sortedids, apks):
             wikidata += " (version code " + app.CurrentVersionCode + ").\n\n"
         validapks = 0
         for apk in apklist:
-            wikidata += "==" + apk['version'] + "==\n"
+            wikidata += "==" + apk['versionName'] + "==\n"
 
             if 'buildproblem' in apk:
                 wikidata += "We can't build this version: " + apk['buildproblem'] + "\n\n"
@@ -209,7 +209,7 @@ def update_wiki(apps, sortedids, apks):
                     wikidata += "F-Droid, and guaranteed to correspond to the source tarball published with it.\n\n"
                 else:
                     wikidata += "the original developer.\n\n"
-            wikidata += "Version code: " + str(apk['versioncode']) + '\n'
+            wikidata += "Version code: " + str(apk['versionCode']) + '\n'
 
         wikidata += '\n[[Category:' + wikicat + ']]\n'
         if len(app.NoSourceSince) > 0:
@@ -494,6 +494,7 @@ def insert_obbs(repodir, apps, apks):
 
     obbs = []
     java_Integer_MIN_VALUE = -pow(2, 31)
+    currentPackageNames = apps.keys()
     for f in glob.glob(os.path.join(repodir, '*.obb')):
         obbfile = os.path.basename(f)
         # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
@@ -504,26 +505,26 @@ def insert_obbs(repodir, apps, apks):
         if not re.match(r'^-?[0-9]+$', chunks[1]):
             obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
             continue
-        versioncode = int(chunks[1])
+        versionCode = int(chunks[1])
         packagename = ".".join(chunks[2:-1])
 
         highestVersionCode = java_Integer_MIN_VALUE
-        if packagename not in apps.keys():
+        if packagename not in currentPackageNames:
             obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
             continue
         for apk in apks:
-            if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
+            if packagename == apk['packageName'] and apk['versionCode'] > highestVersionCode:
-                highestVersionCode = apk['versioncode']
+                highestVersionCode = apk['versionCode']
-        if versioncode > highestVersionCode:
+        if versionCode > highestVersionCode:
-            obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
+            obbWarnDelete(f, 'OBB file has newer versionCode(' + str(versionCode)
                           + ') than any APK: ')
             continue
         obbsha256 = sha256sum(f)
-        obbs.append((packagename, versioncode, obbfile, obbsha256))
+        obbs.append((packagename, versionCode, obbfile, obbsha256))
 
     for apk in apks:
-        for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
+        for (packagename, versionCode, obbfile, obbsha256) in sorted(obbs, reverse=True):
-            if versioncode <= apk['versioncode'] and packagename == apk['id']:
+            if versionCode <= apk['versionCode'] and packagename == apk['packageName']:
                 if obbfile.startswith('main.') and 'obbMainFile' not in apk:
                     apk['obbMainFile'] = obbfile
                     apk['obbMainFileSha256'] = obbsha256
@@ -534,6 +535,98 @@ def insert_obbs(repodir, apps, apks):
                 break
 
 
+def insert_graphics(repodir, apps):
+    """Scans for screenshot PNG files in statically defined screenshots
+    directory and adds them to the app metadata.  The screenshots and
+    graphic must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
+    and must be in the following layout:
+
+    repo/packageName/locale/featureGraphic.png
+    repo/packageName/locale/phoneScreenshots/1.png
+    repo/packageName/locale/phoneScreenshots/2.png
+
+    Where "packageName" is the app's packageName and "locale" is the locale
+    of the graphics, e.g. what language they are in, using the IETF RFC5646
+    format (en-US, fr-CA, es-MX, etc).  This is following this pattern:
+    https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#images-and-screenshots
+
+    This will also scan the metadata/ folder and the apps' source repos
+    for standard locations of graphic and screenshot files.  If it finds
+    them, it will copy them into the repo.
+
+    :param repodir: repo directory to scan
+
+    """
+
+    allowed_extensions = ('png', 'jpg', 'jpeg')
+    graphicnames = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
+    screenshotdirs = ('phoneScreenshots', 'sevenInchScreenshots',
+                      'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
+
+    sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z][A-Z-.@]*'))
+    sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*'))
+
+    for d in sorted(sourcedirs):
+        if not os.path.isdir(d):
+            continue
+        for root, dirs, files in os.walk(d):
+            segments = root.split('/')
+            destdir = os.path.join('repo', segments[1], segments[-1])  # repo/packageName/locale
+            for f in files:
+                base, extension = common.get_extension(f)
+                if base in graphicnames and extension in allowed_extensions:
+                    os.makedirs(destdir, mode=0o755, exist_ok=True)
+                    logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
+                    shutil.copy(os.path.join(root, f), destdir)
+            for d in dirs:
+                if d in screenshotdirs:
+                    for f in glob.glob(os.path.join(root, d, '*.*')):
+                        _, extension = common.get_extension(f)
+                        if extension in allowed_extensions:
+                            screenshotdestdir = os.path.join(destdir, d)
+                            os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
+                            logging.debug('copying ' + f + ' ' + screenshotdestdir)
+                            shutil.copy(f, screenshotdestdir)
+
+    repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
+    for d in repofiles:
+        if not os.path.isdir(d):
+            continue
+        for f in sorted(glob.glob(os.path.join(d, '*.*')) + glob.glob(os.path.join(d, '*Screenshots', '*.*'))):
+            if not os.path.isfile(f):
+                continue
+            segments = f.split('/')
+            packageName = segments[1]
+            locale = segments[2]
+            screenshotdir = segments[3]
+            filename = os.path.basename(f)
+            base, extension = common.get_extension(filename)
+
+            if packageName not in apps:
+                logging.warning('Found "%s" graphic without metadata for app "%s"!'
+                                % (filename, packageName))
+                continue
+            if 'localized' not in apps[packageName]:
+                apps[packageName]['localized'] = collections.OrderedDict()
+            if locale not in apps[packageName]['localized']:
+                apps[packageName]['localized'][locale] = collections.OrderedDict()
+            graphics = apps[packageName]['localized'][locale]
+
+            if extension not in allowed_extensions:
+                logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
+            elif base in graphicnames:
+                # there can only be zero or one of these per locale
+                graphics[base] = filename
+            elif screenshotdir in screenshotdirs:
+                # there can any number of these per locale
+                logging.debug('adding ' + base + ':' + f)
+                if screenshotdir not in graphics:
+                    graphics[screenshotdir] = []
+                graphics[screenshotdir].append(filename)
+            else:
+                logging.warning('Unsupported graphics file found: ' + f)
+
+
 def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
     """Scan a repo for all files with an extension except APK/OBB
 
@@ -565,7 +658,14 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
         usecache = False
         if name in apkcache:
             repo_file = apkcache[name]
-            if repo_file['sha256'] == shasum:
+            # added time is cached as tuple but used here as datetime instance
+            if 'added' in repo_file:
+                a = repo_file['added']
+                if isinstance(a, datetime):
+                    repo_file['added'] = a
+                else:
+                    repo_file['added'] = datetime(*a[:6])
+            if repo_file['hash'] == shasum:
                 logging.debug("Reading " + name + " from cache")
                 usecache = True
             else:
@@ -576,20 +676,21 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
             repo_file = {}
             # TODO rename apkname globally to something more generic
             repo_file['name'] = name
-            repo_file['apkname'] = name
+            repo_file['apkName'] = name
-            repo_file['sha256'] = shasum
+            repo_file['hash'] = shasum
-            repo_file['versioncode'] = 0
+            repo_file['hashType'] = 'sha256'
-            repo_file['version'] = shasum
+            repo_file['versionCode'] = 0
+            repo_file['versionName'] = shasum
             # the static ID is the SHA256 unless it is set in the metadata
-            repo_file['id'] = shasum
+            repo_file['packageName'] = shasum
             n = name.split('_')
             if len(n) == 2:
                 packageName = n[0]
                 versionCode = n[1].split('.')[0]
                 if re.match(r'^-?[0-9]+$', versionCode) \
                    and common.is_valid_package_name(name.split('_')[0]):
-                    repo_file['id'] = packageName
+                    repo_file['packageName'] = packageName
-                    repo_file['versioncode'] = int(versionCode)
+                    repo_file['versionCode'] = int(versionCode)
             srcfilename = name + "_src.tar.gz"
             if os.path.exists(os.path.join(repodir, srcfilename)):
                 repo_file['srcname'] = srcfilename
@@ -605,7 +706,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
             default_date_param = None
 
         # Record in knownapks, getting the added date at the same time..
-        added = knownapks.recordapk(repo_file['apkname'], repo_file['id'],
+        added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
                                     default_date=default_date_param)
         if added:
             repo_file['added'] = added
@@ -661,7 +762,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
         usecache = False
         if apkfilename in apkcache:
             apk = apkcache[apkfilename]
-            if apk['sha256'] == shasum:
+            if apk['hash'] == shasum:
                 logging.debug("Reading " + apkfilename + " from cache")
                 usecache = True
             else:
@@ -670,8 +771,9 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
         if not usecache:
             logging.debug("Processing " + apkfilename)
             apk = {}
-            apk['apkname'] = apkfilename
+            apk['apkName'] = apkfilename
-            apk['sha256'] = shasum
+            apk['hash'] = shasum
+            apk['hashType'] = 'sha256'
             srcfilename = apkfilename[:-4] + "_src.tar.gz"
             if os.path.exists(os.path.join(repodir, srcfilename)):
                 apk['srcname'] = srcfilename
@@ -698,9 +800,9 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
             for line in p.output.splitlines():
                 if line.startswith("package:"):
                     try:
-                        apk['id'] = re.match(name_pat, line).group(1)
+                        apk['packageName'] = re.match(name_pat, line).group(1)
-                        apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
+                        apk['versionCode'] = int(re.match(vercode_pat, line).group(1))
-                        apk['version'] = re.match(vername_pat, line).group(1)
+                        apk['versionName'] = re.match(vername_pat, line).group(1)
                     except Exception as e:
                         logging.error("Package matching failed: " + str(e))
                         logging.info("Line was: " + line)
@@ -814,8 +916,8 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
                                  + 'sudo date -s "' + str(dt_obj) + '"')
 
             iconfilename = "%s.%s.png" % (
-                apk['id'],
+                apk['packageName'],
-                apk['versioncode'])
+                apk['versionCode'])
 
             # Extract the icon file...
             empty_densities = []
@@ -925,12 +1027,12 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
                                 os.path.join(get_icon_dir(repodir, '0'), iconfilename))
 
             if use_date_from_apk and manifest.date_time[1] != 0:
-                default_date_param = datetime(*manifest.date_time).utctimetuple()
+                default_date_param = datetime(*manifest.date_time)
             else:
                 default_date_param = None
 
             # Record in known apks, getting the added date at the same time..
-            added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
+            added = knownapks.recordapk(apk['apkName'], apk['packageName'], default_date=default_date_param)
             if added:
                 apk['added'] = added
 
@@ -1013,6 +1115,181 @@ def make_index(apps, sortedids, apks, repodir, archive):
     :param categories: list of categories
     """
 
+    def _resolve_description_link(appid):
+        if appid in apps:
+            return ("fdroid.app:" + appid, apps[appid].Name)
+        raise MetaDataException("Cannot resolve app id " + appid)
+
+    nosigningkey = False
+    if not options.nosign:
+        if 'repo_keyalias' not in config:
+            nosigningkey = True
+            logging.critical("'repo_keyalias' not found in config.py!")
+        if 'keystore' not in config:
+            nosigningkey = True
+            logging.critical("'keystore' not found in config.py!")
+        if 'keystorepass' not in config and 'keystorepassfile' not in config:
+            nosigningkey = True
+            logging.critical("'keystorepass' not found in config.py!")
+        if 'keypass' not in config and 'keypassfile' not in config:
+            nosigningkey = True
+            logging.critical("'keypass' not found in config.py!")
+        if not os.path.exists(config['keystore']):
+            nosigningkey = True
+            logging.critical("'" + config['keystore'] + "' does not exist!")
+        if nosigningkey:
+            logging.warning("`fdroid update` requires a signing key, you can create one using:")
+            logging.warning("\tfdroid update --create-key")
+            sys.exit(1)
+
+    repodict = collections.OrderedDict()
+    repodict['timestamp'] = datetime.utcnow()
+    repodict['version'] = METADATA_VERSION
+
+    if config['repo_maxage'] != 0:
+        repodict['maxage'] = config['repo_maxage']
+
+    if archive:
+        repodict['name'] = config['archive_name']
+        repodict['icon'] = os.path.basename(config['archive_icon'])
+        repodict['address'] = config['archive_url']
+        repodict['description'] = config['archive_description']
+        urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
+    else:
+        repodict['name'] = config['repo_name']
+        repodict['icon'] = os.path.basename(config['repo_icon'])
+        repodict['address'] = config['repo_url']
+        repodict['description'] = config['repo_description']
+        urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
+
+    mirrorcheckfailed = False
+    mirrors = []
+    for mirror in sorted(config.get('mirrors', [])):
+        base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
+        if config.get('nonstandardwebroot') is not True and base != 'fdroid':
+            logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
+            mirrorcheckfailed = True
+        # must end with / or urljoin strips a whole path segment
+        if mirror.endswith('/'):
+            mirrors.append(urllib.parse.urljoin(mirror, urlbasepath))
+        else:
+            mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath))
+    for mirror in config.get('servergitmirrors', []):
+        mirror = get_raw_mirror(mirror)
+        if mirror is not None:
+            mirrors.append(mirror + '/')
+    if mirrorcheckfailed:
+        sys.exit(1)
+    if mirrors:
+        repodict['mirrors'] = mirrors
+
+    appsWithPackages = collections.OrderedDict()
+    for packageName in sortedids:
+        app = apps[packageName]
+        if app['Disabled']:
+            continue
+
+        # only include apps with packages
+        for apk in apks:
+            if apk['packageName'] == packageName:
+                newapp = copy.copy(app)  # update wiki needs unmodified description
+                newapp['Description'] = metadata.description_html(app['Description'],
+                                                                  _resolve_description_link)
+                appsWithPackages[packageName] = newapp
+                break
+
+    requestsdict = dict()
+    for command in ('install', 'uninstall'):
+        packageNames = []
+        key = command + '_list'
+        if key in config:
+            if isinstance(config[key], str):
+                packageNames = [config[key]]
+            elif all(isinstance(item, str) for item in config[key]):
+                packageNames = config[key]
+            else:
+                raise TypeError('only accepts strings, lists, and tuples')
+        requestsdict[command] = packageNames
+
+    make_index_v0(appsWithPackages, apks, repodir, repodict, requestsdict)
+    make_index_v1(appsWithPackages, apks, repodir, repodict, requestsdict)
+
+
+def make_index_v1(apps, packages, repodir, repodict, requestsdict):
+
+    def _index_encoder_default(obj):
+        if isinstance(obj, set):
+            return list(obj)
+        if isinstance(obj, datetime):
+            return int(obj.timestamp() * 1000)  # Java expects milliseconds
+        raise TypeError(repr(obj) + " is not JSON serializable")
+
+    output = collections.OrderedDict()
+    output['repo'] = repodict
+    output['requests'] = requestsdict
+
+    appslist = []
+    output['apps'] = appslist
+    for appid, appdict in apps.items():
+        d = collections.OrderedDict()
+        appslist.append(d)
+        for k, v in sorted(appdict.items()):
+            if not v:
+                continue
+            if k in ('builds', 'comments', 'metadatapath',
+                     'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes',
+                     'Provides', 'Repo', 'RepoType', 'RequiresRoot',
+                     'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode',
+                     'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'):
+                continue
+
+            # name things after the App class fields in fdroidclient
+            if k == 'id':
+                k = 'packageName'
+            elif k == 'CurrentVersionCode':  # TODO make SuggestedVersionCode the canonical name
+                k = 'suggestedVersionCode'
+            elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
+                k = 'suggestedVersionName'
+            elif k == 'AutoName':
+                if 'Name' not in apps[appid]:
+                    d['name'] = v
+                continue
+            else:
+                k = k[:1].lower() + k[1:]
+            d[k] = v
+
+    output_packages = dict()
+    output['packages'] = output_packages
+    for package in packages:
+        packageName = package['packageName']
+        if packageName in output_packages:
+            packagelist = output_packages[packageName]
+        else:
+            packagelist = []
+            output_packages[packageName] = packagelist
+        d = collections.OrderedDict()
+        packagelist.append(d)
+        for k, v in sorted(package.items()):
+            if not v:
+                continue
+            if k in ('icon', 'icons', 'icons_src', 'name', ):
+                continue
+            d[k] = v
+
+    json_name = 'index-v1.json'
+    index_file = os.path.join(repodir, json_name)
+    with open(index_file, 'w') as fp:
+        json.dump(output, fp, default=_index_encoder_default)
+
+    if options.nosign:
+        logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!')
+    else:
+        common.sign_index_v1(repodir, json_name)
+
+
+def make_index_v0(apps, apks, repodir, repodict, requestsdict):
+    '''aka index.jar aka index.xml'''
+
     doc = Document()
 
     def addElement(name, value, doc, parent):
@@ -1041,92 +1318,29 @@ def make_index(apps, sortedids, apks, repodir, archive):
 
     repoel = doc.createElement("repo")
 
-    mirrorcheckfailed = False
+    repoel.setAttribute("name", repodict['name'])
-    mirrors = []
+    if 'maxage' in repodict:
-    for mirror in sorted(config.get('mirrors', [])):
+        repoel.setAttribute("maxage", str(repodict['maxage']))
-        base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/'))
+    repoel.setAttribute("icon", os.path.basename(repodict['icon']))
-        if config.get('nonstandardwebroot') is not True and base != 'fdroid':
+    repoel.setAttribute("url", repodict['address'])
-            logging.error("mirror '" + mirror + "' does not end with 'fdroid'!")
+    addElement('description', repodict['description'], doc, repoel)
-            mirrorcheckfailed = True
+    for mirror in repodict.get('mirrors', []):
-        # must end with / or urljoin strips a whole path segment
+        addElement('mirror', mirror, doc, repoel)
-        if mirror.endswith('/'):
-            mirrors.append(mirror)
-        else:
-            mirrors.append(mirror + '/')
-    for mirror in config.get('servergitmirrors', []):
-        mirror = get_raw_mirror(mirror)
-        if mirror is not None:
-            mirrors.append(mirror + '/')
-    if mirrorcheckfailed:
-        sys.exit(1)
 
-    if archive:
+    repoel.setAttribute("version", str(repodict['version']))
-        repoel.setAttribute("name", config['archive_name'])
+    repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp())
-        if config['repo_maxage'] != 0:
-            repoel.setAttribute("maxage", str(config['repo_maxage']))
-        repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
-        repoel.setAttribute("url", config['archive_url'])
-        addElement('description', config['archive_description'], doc, repoel)
-        urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path)
-        for mirror in mirrors:
-            addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
-
-    else:
-        repoel.setAttribute("name", config['repo_name'])
-        if config['repo_maxage'] != 0:
-            repoel.setAttribute("maxage", str(config['repo_maxage']))
-        repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
-        repoel.setAttribute("url", config['repo_url'])
-        addElement('description', config['repo_description'], doc, repoel)
-        urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path)
-        for mirror in mirrors:
-            addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
-
-    repoel.setAttribute("version", str(METADATA_VERSION))
-    repoel.setAttribute("timestamp", str(int(time.time())))
-
-    nosigningkey = False
-    if not options.nosign:
-        if 'repo_keyalias' not in config:
-            nosigningkey = True
-            logging.critical("'repo_keyalias' not found in config.py!")
-        if 'keystore' not in config:
-            nosigningkey = True
-            logging.critical("'keystore' not found in config.py!")
-        if 'keystorepass' not in config and 'keystorepassfile' not in config:
-            nosigningkey = True
-            logging.critical("'keystorepass' not found in config.py!")
-        if 'keypass' not in config and 'keypassfile' not in config:
-            nosigningkey = True
-            logging.critical("'keypass' not found in config.py!")
-        if not os.path.exists(config['keystore']):
-            nosigningkey = True
-            logging.critical("'" + config['keystore'] + "' does not exist!")
-        if nosigningkey:
-            logging.warning("`fdroid update` requires a signing key, you can create one using:")
-            logging.warning("\tfdroid update --create-key")
-            sys.exit(1)
 
     repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
     root.appendChild(repoel)
 
     for command in ('install', 'uninstall'):
-        packageNames = []
+        for packageName in requestsdict[command]:
-        key = command + '_list'
-        if key in config:
-            if isinstance(config[key], str):
-                packageNames = [config[key]]
-            elif all(isinstance(item, str) for item in config[key]):
-                packageNames = config[key]
-            else:
-                raise TypeError('only accepts strings, lists, and tuples')
-        for packageName in packageNames:
             element = doc.createElement(command)
             root.appendChild(element)
             element.setAttribute('packageName', packageName)
 
-    for appid in sortedids:
+    for appid, appdict in apps.items():
-        app = metadata.App(apps[appid])
+        app = metadata.App(appdict)
 
         if app.Disabled is not None:
             continue
@@ -1134,7 +1348,7 @@ def make_index(apps, sortedids, apks, repodir, archive):
         # Get a list of the apks for this app...
         apklist = []
         for apk in apks:
-            if apk['id'] == appid:
+            if apk['packageName'] == appid:
                 apklist.append(apk)
 
         if len(apklist) == 0:
@@ -1146,22 +1360,19 @@ def make_index(apps, sortedids, apks, repodir, archive):
 
         addElement('id', app.id, doc, apel)
         if app.added:
-            addElement('added', time.strftime('%Y-%m-%d', app.added), doc, apel)
+            addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel)
         if app.lastUpdated:
-            addElement('lastupdated', time.strftime('%Y-%m-%d', app.lastUpdated), doc, apel)
+            addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel)
         addElement('name', app.Name, doc, apel)
         addElement('summary', app.Summary, doc, apel)
         if app.icon:
             addElement('icon', app.icon, doc, apel)
 
-        def linkres(appid):
+        if app.get('Description'):
-            if appid in apps:
+            description = app.Description
-                return ("fdroid.app:" + appid, apps[appid].Name)
+        else:
-            raise MetaDataException("Cannot resolve app id " + appid)
+            description = '<p>No description available</p>'
-
+        addElement('desc', description, doc, apel)
-        addElement('desc',
-                   metadata.description_html(app.Description, linkres),
-                   doc, apel)
         addElement('license', app.License, doc, apel)
         if app.Categories:
             addElement('categories', ','.join(app.Categories), doc, apel)
@@ -1194,7 +1405,7 @@ def make_index(apps, sortedids, apks, repodir, archive):
 
         # Sort the apk list into version order, just so the web site
         # doesn't have to do any work by default...
-        apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
+        apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True)
 
         if 'antiFeatures' in apklist[0]:
             app.AntiFeatures.extend(apklist[0]['antiFeatures'])
@@ -1203,34 +1414,33 @@ def make_index(apps, sortedids, apks, repodir, archive):
 
         # Check for duplicates - they will make the client unhappy...
         for i in range(len(apklist) - 1):
-            if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
+            if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']:
                 logging.critical("duplicate versions: '%s' - '%s'" % (
-                    apklist[i]['apkname'], apklist[i + 1]['apkname']))
+                    apklist[i]['apkName'], apklist[i + 1]['apkName']))
                 sys.exit(1)
 
         current_version_code = 0
         current_version_file = None
         for apk in apklist:
-            file_extension = common.get_file_extension(apk['apkname'])
+            file_extension = common.get_file_extension(apk['apkName'])
             # find the APK for the "Current Version"
-            if current_version_code < apk['versioncode']:
+            if current_version_code < apk['versionCode']:
-                current_version_code = apk['versioncode']
+                current_version_code = apk['versionCode']
             if current_version_code < int(app.CurrentVersionCode):
-                current_version_file = apk['apkname']
+                current_version_file = apk['apkName']
 
             apkel = doc.createElement("package")
             apel.appendChild(apkel)
-            addElement('version', apk['version'], doc, apkel)
+            addElement('version', apk['versionName'], doc, apkel)
-            addElement('versioncode', str(apk['versioncode']), doc, apkel)
+            addElement('versioncode', str(apk['versionCode']), doc, apkel)
-            addElement('apkname', apk['apkname'], doc, apkel)
+            addElement('apkname', apk['apkName'], doc, apkel)
             addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
-            for hash_type in ['sha256']:
+
-                if hash_type not in apk:
+            hashel = doc.createElement("hash")
-                    continue
+            hashel.setAttribute('type', 'sha256')
-                hashel = doc.createElement("hash")
+            hashel.appendChild(doc.createTextNode(apk['hash']))
-                hashel.setAttribute("type", hash_type)
+            apkel.appendChild(hashel)
-                hashel.appendChild(doc.createTextNode(apk[hash_type]))
+
-                apkel.appendChild(hashel)
             addElement('size', str(apk['size']), doc, apkel)
             addElementIfInApk('sdkver', apk,
                               'minSdkVersion', doc, apkel)
@@ -1247,7 +1457,7 @@ def make_index(apps, sortedids, apks, repodir, archive):
             addElementIfInApk('obbPatchFileSha256', apk,
                               'obbPatchFileSha256', doc, apkel)
             if 'added' in apk:
-                addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
+                addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel)
 
             if file_extension == 'apk':  # sig is required for APKs, but only APKs
                 addElement('sig', apk['sig'], doc, apkel)
@@ -1326,18 +1536,7 @@ def make_index(apps, sortedids, apks, repodir, archive):
             if os.path.exists(signed):
                 os.remove(signed)
         else:
-            args = [config['jarsigner'], '-keystore', config['keystore'],
+            common.signjar(signed)
-                    '-storepass:file', config['keystorepassfile'],
-                    '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
-                    signed, config['repo_keyalias']]
-            if config['keystore'] == 'NONE':
-                args += config['smartcardoptions']
-            else:  # smardcards never use -keypass
-                args += ['-keypass:file', config['keypassfile']]
-            p = FDroidPopen(args)
-            if p.returncode != 0:
-                logging.critical("Failed to sign index")
-                sys.exit(1)
 
     # Copy the repo icon into the repo directory...
     icon_dir = os.path.join(repodir, 'icons')
@@ -1366,11 +1565,11 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
         def filter_apk_list_sorted(apk_list):
             res = []
             for apk in apk_list:
-                if apk['id'] == appid:
+                if apk['packageName'] == appid:
                     res.append(apk)
 
             # Sort the apk list by version code. First is highest/newest.
-            return sorted(res, key=lambda apk: apk['versioncode'], reverse=True)
+            return sorted(res, key=lambda apk: apk['versionCode'], reverse=True)
 
         def move_file(from_dir, to_dir, filename, ignore_missing):
             from_path = os.path.join(from_dir, filename)
@@ -1386,9 +1585,9 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
             apklist = filter_apk_list_sorted(apks)
             # Move back the ones we don't want.
             for apk in apklist[keepversions:]:
-                logging.info("Moving " + apk['apkname'] + " to archive")
+                logging.info("Moving " + apk['apkName'] + " to archive")
-                move_file(repodir, archivedir, apk['apkname'], False)
+                move_file(repodir, archivedir, apk['apkName'], False)
-                move_file(repodir, archivedir, apk['apkname'] + '.asc', True)
+                move_file(repodir, archivedir, apk['apkName'] + '.asc', True)
                 for density in all_screen_densities:
                     repo_icon_dir = get_icon_dir(repodir, density)
                     archive_icon_dir = get_icon_dir(archivedir, density)
@@ -1404,9 +1603,9 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
             archapklist = filter_apk_list_sorted(archapks)
             # Move forward the ones we want again.
             for apk in archapklist[:required]:
-                logging.info("Moving " + apk['apkname'] + " from archive")
+                logging.info("Moving " + apk['apkName'] + " from archive")
-                move_file(archivedir, repodir, apk['apkname'], False)
+                move_file(archivedir, repodir, apk['apkName'], False)
-                move_file(archivedir, repodir, apk['apkname'] + '.asc', True)
+                move_file(archivedir, repodir, apk['apkName'] + '.asc', True)
                 for density in all_screen_densities:
                     repo_icon_dir = get_icon_dir(repodir, density)
                     archive_icon_dir = get_icon_dir(archivedir, density)
@@ -1422,16 +1621,16 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi
 def add_apks_to_per_app_repos(repodir, apks):
     apks_per_app = dict()
     for apk in apks:
-        apk['per_app_dir'] = os.path.join(apk['id'], 'fdroid')
+        apk['per_app_dir'] = os.path.join(apk['packageName'], 'fdroid')
         apk['per_app_repo'] = os.path.join(apk['per_app_dir'], 'repo')
         apk['per_app_icons'] = os.path.join(apk['per_app_repo'], 'icons')
-        apks_per_app[apk['id']] = apk
+        apks_per_app[apk['packageName']] = apk
 
         if not os.path.exists(apk['per_app_icons']):
-            logging.info('Adding new repo for only ' + apk['id'])
+            logging.info('Adding new repo for only ' + apk['packageName'])
             os.makedirs(apk['per_app_icons'])
 
-        apkpath = os.path.join(repodir, apk['apkname'])
+        apkpath = os.path.join(repodir, apk['apkName'])
         shutil.copy(apkpath, apk['per_app_repo'])
         apksigpath = apkpath + '.sig'
         if os.path.exists(apksigpath):
@@ -1475,11 +1674,11 @@ def make_binary_transparency_log(repodirs):
         cpdir = os.path.join(btrepo, repodir)
         if not os.path.exists(cpdir):
             os.mkdir(cpdir)
-        for f in ('index.xml', ):
+        for f in ('index.xml', 'index-v1.json'):
             dest = os.path.join(cpdir, f)
             shutil.copyfile(os.path.join(repodir, f), dest)
             gitrepo.index.add([os.path.join(repodir, f), ])
-        for f in ('index.jar', ):
+        for f in ('index.jar', 'index-v1.jar'):
             repof = os.path.join(repodir, f)
             dest = os.path.join(cpdir, f)
             jarin = zipfile.ZipFile(repof, 'r')
@@ -1639,12 +1838,12 @@ def main():
     # metadata files, if requested on the command line)
     newmetadata = False
     for apk in apks:
-        if apk['id'] not in apps:
+        if apk['packageName'] not in apps:
             if options.create_metadata:
                 if 'name' not in apk:
-                    logging.error(apk['id'] + ' does not have a name! Skipping...')
+                    logging.error(apk['packageName'] + ' does not have a name! Skipping...')
                     continue
-                f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
+                f = open(os.path.join('metadata', apk['packageName'] + '.txt'), 'w', encoding='utf8')
                 f.write("License:Unknown\n")
                 f.write("Web Site:\n")
                 f.write("Source Code:\n")
@@ -1656,13 +1855,13 @@ def main():
                 f.write(".\n")
                 f.write("Name:" + apk['name'] + "\n")
                 f.close()
-                logging.info("Generated skeleton metadata for " + apk['id'])
+                logging.info("Generated skeleton metadata for " + apk['packageName'])
                 newmetadata = True
             else:
-                msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
+                msg = apk['apkName'] + " (" + apk['packageName'] + ") has no metadata!"
                 if options.delete_unknown:
-                    logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
+                    logging.warn(msg + "\n\tdeleting: repo/" + apk['apkName'])
-                    rmf = os.path.join(repodirs[0], apk['apkname'])
+                    rmf = os.path.join(repodirs[0], apk['apkName'])
                     if not os.path.exists(rmf):
                         logging.error("Could not find {0} to remove it".format(rmf))
                     else:
@@ -1675,6 +1874,7 @@ def main():
         apps = metadata.read_metadata()
 
     insert_obbs(repodirs[0], apps, apks)
+    insert_graphics(repodirs[0], apps)
 
     # Scan the archive repo for apks as well
     if len(repodirs) > 1:
@@ -1694,9 +1894,9 @@ def main():
     for appid, app in apps.items():
         bestver = UNSET_VERSION_CODE
         for apk in apks + archapks:
-            if apk['id'] == appid:
+            if apk['packageName'] == appid:
-                if apk['versioncode'] > bestver:
+                if apk['versionCode'] > bestver:
-                    bestver = apk['versioncode']
+                    bestver = apk['versionCode']
                     bestapk = apk
 
                 if 'added' in apk:

fdroidserver/verify.py

@@ -37,7 +37,7 @@ def main():
     # Parse command line...
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
-    parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    parser.add_argument("appid", nargs='*', help="app-id with optional versionCode in the form APPID[:VERCODE]")
     options = parser.parse_args()
 
     config = common.read_config(options)

tests/common.TestCase

@@ -157,6 +157,26 @@ class CommonTest(unittest.TestCase):
         p = fdroidserver.common.FDroidPopen(commands, stderr_to_stdout=False)
         self.assertEqual(p.output, 'stdout message\n')
 
+    def test_signjar(self):
+        fdroidserver.common.config = None
+        config = fdroidserver.common.read_config(fdroidserver.common.options)
+        config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
+        fdroidserver.common.config = config
+
+        basedir = os.path.dirname(__file__)
+        tmpdir = os.path.join(basedir, '..', '.testfiles')
+        if not os.path.exists(tmpdir):
+            os.makedirs(tmpdir)
+        sourcedir = os.path.join(basedir, 'signindex')
+        testsdir = tempfile.mkdtemp(prefix='test_signjar', dir=tmpdir)
+        for f in ('testy.jar', 'guardianproject.jar',):
+            sourcefile = os.path.join(sourcedir, f)
+            testfile = os.path.join(testsdir, f)
+            shutil.copy(sourcefile, testsdir)
+            fdroidserver.common.signjar(testfile)
+            # these should be resigned, and therefore different
+            self.assertNotEqual(open(sourcefile, 'rb').read(), open(testfile, 'rb').read())
+
 
 if __name__ == "__main__":
     parser = optparse.OptionParser()

tests/metadata/info.guardianproject.urzip.txt

@@ -1,25 +0,0 @@
-Categories:Development,GuardianProject
-License:GPLv3
-Web Site:https://dev.guardianproject.info/projects/urzip
-Source Code:https://github.com/guardianproject/urzip
-Issue Tracker:https://dev.guardianproject.info/projects/urzip/issues
-Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
-
-Auto Name:Urzip：本地应用的信息
-Summary:一个实用工具，获取已安装在您的设备上的应用的有关信息
-Description:
-It’s Urzip 是一个获得已安装 APK 相关信息的实用工具。它从您的设备上已安装的所有应用开始，一键触摸即可显示 APK 的指纹，并且提供到达 virustotal.com 和 androidobservatory.org 的快捷链接，让您方便地了解特定 APK 的档案。它还可以让您导出签名证书和生成 ApkSignaturePin Pin 文件供 TrustedIntents 库使用。
-
-★ Urzip 支持下列语言： Deutsch, English, español, suomi, 日本語, 한국어, Norsk, português (Portugal), Русский, Slovenščina, Türkçe
-没看到您的语言？帮忙翻译本应用吧：
-https://www.transifex.com/projects/p/urzip
-
-★ 致用户：我们还缺少你喜欢的功能？发现了一个 bug？请告诉我们！我们乐于听取您的意见。请发送电子邮件至: support@guardianproject.info 或者加入我们的聊天室 https://guardianproject.info/contact
-.
-
-Repo Type:git
-Repo:https://github.com/guardianproject/urzip.git
-
-
-
-Current Version Code:9999999

tests/metadata/info.guardianproject.urzip.yml

@@ -0,0 +1,26 @@
+AutoName: Urzip：本地应用的信息
+AutoUpdateMode: None
+Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+Categories:
+- Development
+- GuardianProject
+- 1
+- 2.0
+CurrentVersionCode: 2147483647
+Description: |
+  It’s Urzip 是一个获得已安装 APK 相关信息的实用工具。它从您的设备上已安装的所有应用开始，一键触摸即可显示 APK 的指纹，并且提供到达 virustotal.com 和 androidobservatory.org 的快捷链接，让您方便地了解特定 APK 的档案。它还可以让您导出签名证书和生成 ApkSignaturePin Pin 文件供 TrustedIntents 库使用。
+
+  ★ Urzip 支持下列语言： Deutsch, English, español, suomi, 日本語, 한국어, Norsk, português (Portugal), Русский, Slovenščina, Türkçe
+  没看到您的语言？帮忙翻译本应用吧：
+  https://www.transifex.com/projects/p/urzip
+
+  ★ 致用户：我们还缺少你喜欢的功能？发现了一个 bug？请告诉我们！我们乐于听取您的意见。请发送电子邮件至: support@guardianproject.info 或者加入我们的聊天室 https://guardianproject.info/contact
+
+IssueTracker: https://dev.guardianproject.info/projects/urzip/issues
+License: GPLv3
+Repo: https://github.com/guardianproject/urzip.git
+RepoType: git
+SourceCode: https://github.com/guardianproject/urzip
+Summary: 一个实用工具，获取已安装在您的设备上的应用的有关信息
+UpdateCheckMode: None
+WebSite: https://dev.guardianproject.info/projects/urzip

tests/repo/categories.txt

@@ -1,3 +1,5 @@
+1
+2.0
 Development
 GuardianProject
 Multimedia

tests/repo/index.xml

@@ -164,14 +164,14 @@
 		<icon>info.guardianproject.urzip.100.png</icon>
 		<desc>&lt;p&gt;It’s Urzip 是一个获得已安装 APK 相关信息的实用工具。它从您的设备上已安装的所有应用开始，一键触摸即可显示 APK 的指纹，并且提供到达 virustotal.com 和 androidobservatory.org 的快捷链接，让您方便地了解特定 APK 的档案。它还可以让您导出签名证书和生成 ApkSignaturePin Pin 文件供 TrustedIntents 库使用。&lt;/p&gt;&lt;p&gt;★ Urzip 支持下列语言： Deutsch, English, español, suomi, 日本語, 한국어, Norsk, português (Portugal), Русский, Slovenščina, Türkçe 没看到您的语言？帮忙翻译本应用吧： https://www.transifex.com/projects/p/urzip&lt;/p&gt;&lt;p&gt;★ 致用户：我们还缺少你喜欢的功能？发现了一个 bug？请告诉我们！我们乐于听取您的意见。请发送电子邮件至: support@guardianproject.info 或者加入我们的聊天室 https://guardianproject.info/contact&lt;/p&gt;</desc>
 		<license>GPLv3</license>
-		<categories>Development,GuardianProject</categories>
+		<categories>Development,GuardianProject,1,2.0</categories>
 		<category>Development</category>
 		<web>https://dev.guardianproject.info/projects/urzip</web>
 		<source>https://github.com/guardianproject/urzip</source>
 		<tracker>https://dev.guardianproject.info/projects/urzip/issues</tracker>
 		<bitcoin>1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk</bitcoin>
 		<marketversion></marketversion>
-		<marketvercode>9999999</marketvercode>
+		<marketvercode>2147483647</marketvercode>
 		<package>
 			<version>0.1</version>
 			<versioncode>100</versioncode>

tests/run-tests

@@ -113,6 +113,33 @@ echo_header "print fdroid version"
 $fdroid --version
 
 
+#------------------------------------------------------------------------------#
+echo_header 'run process when building and signing are on separate machines'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+cp $WORKSPACE/tests/keystore.jks $REPOROOT/
+$fdroid init --keystore keystore.jks --repo-keyalias=sova
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo "accepted_formats = ['txt', 'yml']" >> config.py
+echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py
+test -d archive || mkdir archive
+test -d metadata || mkdir metadata
+cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/
+test -d repo || mkdir repo
+test -d unsigned || mkdir unsigned
+cp $WORKSPACE/tests/urzip-release-unsigned.apk unsigned/info.guardianproject.urzip_100.apk
+$fdroid publish --verbose
+$fdroid update --verbose --nosign
+$fdroid signindex --verbose
+test -e repo/index.xml
+test -e repo/index.jar
+test -e repo/index-v1.jar
+test -L urzip.apk
+grep -F '<application id=' repo/index.xml > /dev/null
+
+
 #------------------------------------------------------------------------------#
 echo_header "test UTF-8 metadata"
 
@@ -124,7 +151,7 @@ sed -i.tmp 's,^ *repo_description.*,repo_description = """获取已安装在您
 echo "mirrors = ('https://foo.bar/fdroid', 'http://secret.onion/fdroid')" >> config.py
 mkdir metadata
 cp $WORKSPACE/tests/urzip.apk repo/
-cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.txt metadata/
+cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/
 
 $fdroid readmeta
 $fdroid update
@@ -169,6 +196,7 @@ echo "mirrors = ('http://foobarfoobarfoobar.onion/fdroid','https://foo.bar/fdroi
 $fdroid update --verbose --pretty
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 grep -F '<install packageName=' repo/index.xml > /dev/null
 grep -F '<uninstall packageName=' repo/index.xml > /dev/null
@@ -424,6 +452,7 @@ $fdroid readmeta
 grep -F '<application id=' repo/index.xml > /dev/null
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 export ANDROID_HOME=$STORED_ANDROID_HOME
 
 
@@ -453,6 +482,7 @@ $fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -481,6 +511,7 @@ $fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -497,6 +528,7 @@ $fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 test -e $REPOROOT/repo/info.guardianproject.urzip_100.apk || \
     cp $WORKSPACE/tests/urzip.apk $REPOROOT/repo/
@@ -504,6 +536,7 @@ $fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 
 
@@ -569,6 +602,7 @@ echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
 $fdroid update --verbose --pretty
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 cd binary_transparency
 [ `git rev-list --count HEAD` == "2" ]
@@ -587,6 +621,7 @@ $fdroid update --create-metadata --verbose
 $fdroid readmeta
 test -e repo/index.xml
 test -e repo/index.jar
+test -e repo/index-v1.jar
 grep -F '<application id=' repo/index.xml > /dev/null
 
 # now set fake repo_keyalias

tests/update.TestCase

@@ -110,23 +110,23 @@ class UpdateTest(unittest.TestCase):
 
         fdroidserver.update.insert_obbs('repo', apps, apks)
         for apk in apks:
-            if apk['id'] == 'obb.mainpatch.current':
+            if apk['packageName'] == 'obb.mainpatch.current':
                 self.assertEqual(apk.get('obbMainFile'), 'main.1619.obb.mainpatch.current.obb')
                 self.assertEqual(apk.get('obbPatchFile'), 'patch.1619.obb.mainpatch.current.obb')
-            elif apk['id'] == 'obb.main.oldversion':
+            elif apk['packageName'] == 'obb.main.oldversion':
                 self.assertEqual(apk.get('obbMainFile'), 'main.1434483388.obb.main.oldversion.obb')
                 self.assertIsNone(apk.get('obbPatchFile'))
-            elif apk['id'] == 'obb.main.twoversions':
+            elif apk['packageName'] == 'obb.main.twoversions':
                 self.assertIsNone(apk.get('obbPatchFile'))
-                if apk['versioncode'] == 1101613:
+                if apk['versionCode'] == 1101613:
                     self.assertEqual(apk.get('obbMainFile'), 'main.1101613.obb.main.twoversions.obb')
-                elif apk['versioncode'] == 1101615:
+                elif apk['versionCode'] == 1101615:
                     self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
-                elif apk['versioncode'] == 1101617:
+                elif apk['versionCode'] == 1101617:
                     self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
                 else:
                     self.assertTrue(False)
-            elif apk['id'] == 'info.guardianproject.urzip':
+            elif apk['packageName'] == 'info.guardianproject.urzip':
                 self.assertIsNone(apk.get('obbMainFile'))
                 self.assertIsNone(apk.get('obbPatchFile'))