Merge branch 'support-media-files' into 'master'

Support media files

This is the first step in supporting adding artibitrary files to fdroid repos, targeted add supporting media files and flashable _update.zip_ files like used to install the Privileged Extension.  This reuses the existing metadata fields to keep compatibilty with older versions, but that means that lots of names are confusingly named since they refer to aspects of an APK rather than a generic file.  We can address that later when we refactor the whole index metadata.

See merge request !177

fdroidserver/common.py

@@ -1609,6 +1609,12 @@ class KnownApks:
         return lst
 
 
+def get_file_extension(filename):
+    """get the normalized file extension, can be blank string but never None"""
+
+    return os.path.splitext(filename)[1].lower()[1:]
+
+
 def isApkDebuggable(apkfile, config):
     """Returns True if the given apk file is debuggable
 

fdroidserver/update.py

@@ -504,12 +504,82 @@ def insert_obbs(repodir, apps, apks):
                 break
 
 
-def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
+def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
+    """Scan a repo for all files with an extension except APK/OBB
+
+    :param apkcache: current cached info about all repo files
+    :param repodir: repo directory to scan
+    :param knownapks: list of all known files, as per metadata.read_metadata
+    :param use_date_from_file: use date from file (instead of current date)
+                               for newly added files
+    """
+
+    cachechanged = False
+    repo_files = []
+    for name in os.listdir(repodir):
+        if name in ['index.jar', 'index.xml', 'index.html', 'categories.txt', ]:
+            continue
+        file_extension = common.get_file_extension(name)
+        if file_extension == 'apk' or file_extension == 'obb':
+            continue
+        filename = os.path.join(repodir, name)
+        if not os.path.isfile(filename):
+            continue
+        stat = os.stat(filename)
+        if stat.st_size == 0:
+            logging.error(filename + ' is zero size!')
+            sys.exit(1)
+
+        shasum = sha256sum(filename)
+        usecache = False
+        if name in apkcache:
+            repo_file = apkcache[name]
+            if repo_file['sha256'] == shasum:
+                logging.debug("Reading " + name + " from cache")
+                usecache = True
+            else:
+                logging.debug("Ignoring stale cache data for " + name)
+        elif not usecache:
+            logging.debug("Processing " + name)
+            repo_file = {}
+            # TODO rename apkname globally to something more generic
+            repo_file['name'] = name
+            repo_file['apkname'] = name
+            repo_file['sha256'] = shasum
+            repo_file['versioncode'] = 0
+            repo_file['version'] = shasum
+            # the static ID is the SHA256 unless it is set in the metadata
+            repo_file['id'] = shasum
+            srcfilename = name + ".src.tar.gz"
+            if os.path.exists(os.path.join(repodir, srcfilename)):
+                repo_file['srcname'] = srcfilename
+            repo_file['size'] = stat.st_size
+
+            apkcache[name] = repo_file
+            cachechanged = True
+
+        if use_date_from_file:
+            timestamp = stat.st_ctime
+            default_date_param = datetime.fromtimestamp(timestamp).utctimetuple()
+        else:
+            default_date_param = None
+
+        # Record in knownapks, getting the added date at the same time..
+        added = knownapks.recordapk(repo_file['apkname'], repo_file['id'],
+                                    default_date=default_date_param)
+        if added:
+            repo_file['added'] = added
+
+        repo_files.append(repo_file)
+
+    return repo_files, cachechanged
+
+
+def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
     """Scan the apks in the given repo directory.
 
     This also extracts the icons.
 
-    :param apps: list of all applications, as per metadata.read_metadata
     :param apkcache: current apk cache information
     :param repodir: repo directory to scan
     :param knownapks: known apks info
@@ -884,6 +954,12 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
             return
         addElement(name, value, doc, parent)
 
+    def addElementIfInApk(name, apk, key, doc, parent):
+        if key not in apk:
+            return
+        value = str(apk[key])
+        addElement(name, value, doc, parent)
+
     def addElementCDATA(name, value, doc, parent):
         el = doc.createElement(name)
         el.appendChild(doc.createCDATASection(value))
@@ -1059,6 +1135,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
         current_version_code = 0
         current_version_file = None
         for apk in apklist:
+            file_extension = common.get_file_extension(apk['apkname'])
             # find the APK for the "Current Version"
             if current_version_code < apk['versioncode']:
                 current_version_code = apk['versioncode']
@@ -1070,8 +1147,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
             addElement('version', apk['version'], doc, apkel)
             addElement('versioncode', str(apk['versioncode']), doc, apkel)
             addElement('apkname', apk['apkname'], doc, apkel)
-            if 'srcname' in apk:
+            addElementIfInApk('srcname', apk, 'srcname', doc, apkel)
-                addElement('srcname', apk['srcname'], doc, apkel)
             for hash_type in ['sha256']:
                 if hash_type not in apk:
                     continue
@@ -1079,44 +1155,49 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
                 hashel.setAttribute("type", hash_type)
                 hashel.appendChild(doc.createTextNode(apk[hash_type]))
                 apkel.appendChild(hashel)
-            addElement('sig', apk['sig'], doc, apkel)
             addElement('size', str(apk['size']), doc, apkel)
             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
-            if 'targetSdkVersion' in apk:
+            addElementIfInApk('targetSdkVersion', apk,
-                addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
+                              'targetSdkVersion', doc, apkel)
-            if 'maxSdkVersion' in apk:
+            addElementIfInApk('maxsdkver', apk,
-                addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
+                              'maxSdkVersion', doc, apkel)
-            addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
+            addElementIfInApk('obbMainFile', apk,
-            addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
+                              'obbMainFile', doc, apkel)
-            addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
+            addElementIfInApk('obbMainFileSha256', apk,
-            addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
+                              'obbMainFileSha256', doc, apkel)
+            addElementIfInApk('obbPatchFile', apk,
+                              'obbPatchFile', doc, apkel)
+            addElementIfInApk('obbPatchFileSha256', apk,
+                              'obbPatchFileSha256', doc, apkel)
             if 'added' in apk:
                 addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
 
-            # TODO: remove old permission format
+            if file_extension == 'apk':  # sig is required for APKs, but only APKs
-            old_permissions = set()
+                addElement('sig', apk['sig'], doc, apkel)
-            for perm in apk['uses-permission']:
-                perm_name = perm.name
-                if perm_name.startswith("android.permission."):
-                    perm_name = perm_name[19:]
-                old_permissions.add(perm_name)
-            addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
 
-            for permission in apk['uses-permission']:
+                old_permissions = set()
-                permel = doc.createElement('uses-permission')
+                for perm in apk['uses-permission']:
-                permel.setAttribute('name', permission.name)
+                    perm_name = perm.name
-                if permission.maxSdkVersion is not None:
+                    if perm_name.startswith("android.permission."):
-                    permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
+                        perm_name = perm_name[19:]
-                    apkel.appendChild(permel)
+                    old_permissions.add(perm_name)
-            for permission_sdk_23 in apk['uses-permission-sdk-23']:
+                addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
-                permel = doc.createElement('uses-permission-sdk-23')
+
-                permel.setAttribute('name', permission_sdk_23.name)
+                for permission in apk['uses-permission']:
-                if permission_sdk_23.maxSdkVersion is not None:
+                    permel = doc.createElement('uses-permission')
-                    permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
+                    permel.setAttribute('name', permission.name)
-                    apkel.appendChild(permel)
+                    if permission.maxSdkVersion is not None:
-            if 'nativecode' in apk:
+                        permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
-                addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
+                        apkel.appendChild(permel)
-            addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
+                for permission_sdk_23 in apk['uses-permission-sdk-23']:
+                    permel = doc.createElement('uses-permission-sdk-23')
+                    permel.setAttribute('name', permission_sdk_23.name)
+                    if permission_sdk_23.maxSdkVersion is not None:
+                        permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
+                        apkel.appendChild(permel)
+                if 'nativecode' in apk:
+                    addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
+                addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
 
         if current_version_file is not None \
                 and config['make_current_version_link'] \
@@ -1394,8 +1475,12 @@ def main():
     delete_disabled_builds(apps, apkcache, repodirs)
 
     # Scan all apks in the main repo
-    apks, cachechanged = scan_apks(apps, apkcache, repodirs[0], knownapks, options.use_date_from_apk)
+    apks, cachechanged = scan_apks(apkcache, repodirs[0], knownapks, options.use_date_from_apk)
 
+    files, fcachechanged = scan_repo_files(apkcache, repodirs[0], knownapks,
+                                           options.use_date_from_apk)
+    cachechanged = cachechanged or fcachechanged
+    apks += files
     # Generate warnings for apk's with no metadata (or create skeleton
     # metadata files, if requested on the command line)
     newmetadata = False
@@ -1438,18 +1523,21 @@ def main():
 
     # Scan the archive repo for apks as well
     if len(repodirs) > 1:
-        archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
+        archapks, cc = scan_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
         if cc:
             cachechanged = True
     else:
         archapks = []
 
+    # less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
+    UNSET_VERSION_CODE = -0x100000000
+
     # Some information from the apks needs to be applied up to the application
     # level. When doing this, we use the info from the most recent version's apk.
     # We deal with figuring out when the app was added and last updated at the
     # same time.
     for appid, app in apps.items():
-        bestver = 0
+        bestver = UNSET_VERSION_CODE
         for apk in apks + archapks:
             if apk['id'] == appid:
                 if apk['versioncode'] > bestver:
@@ -1467,7 +1555,7 @@ def main():
         if not app.lastupdated:
             logging.debug("Don't know when " + appid + " was last updated")
 
-        if bestver == 0:
+        if bestver == UNSET_VERSION_CODE:
             if app.Name is None:
                 app.Name = app.AutoName or appid
             app.icon = None

tests/update.TestCase

@@ -101,7 +101,7 @@ class UpdateTest(unittest.TestCase):
 
         apps = fdroidserver.metadata.read_metadata(xref=True)
         knownapks = fdroidserver.common.KnownApks()
-        apks, cachechanged = fdroidserver.update.scan_apks(apps, {}, 'repo', knownapks, False)
+        apks, cachechanged = fdroidserver.update.scan_apks({}, 'repo', knownapks, False)
         self.assertEqual(len(apks), 6)
         apk = apks[0]
         self.assertEqual(apk['minSdkVersion'], '4')