Merge branch 'miscfixes' into 'master'

misc fixes

Closes #268

See merge request !245

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: fdroid/ci:server-20161223
+image: registry.gitlab.com/fdroid/ci-images:server-latest
 
 test:
   script:

buildserver/provision-android-sdk

@@ -82,6 +82,8 @@ echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;sup
 echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.2"
 
 chmod -R a+rX $ANDROID_HOME/
+chgrp vagrant $ANDROID_HOME
+chmod g+w $ANDROID_HOME
 find $ANDROID_HOME/ -type f -executable -print0 | xargs -0 chmod a+x
 
 # allow gradle to install newer build-tools versions

fdroidserver/common.py

@@ -1700,7 +1700,8 @@ class KnownApks:
 
 def get_file_extension(filename):
     """get the normalized file extension, can be blank string but never None"""
-
+    if isinstance(filename, bytes):
+        filename = filename.decode('utf-8')
     return os.path.splitext(filename)[1].lower()[1:]
 
 
@@ -2333,15 +2334,17 @@ def get_per_app_repos():
 
 def is_repo_file(filename):
     '''Whether the file in a repo is a build product to be delivered to users'''
+    if isinstance(filename, str):
+        filename = filename.encode('utf-8', errors="surrogateescape")
     return os.path.isfile(filename) \
-        and not filename.endswith('.asc') \
+        and not filename.endswith(b'.asc') \
-        and not filename.endswith('.sig') \
+        and not filename.endswith(b'.sig') \
         and os.path.basename(filename) not in [
-            'index.jar',
+            b'index.jar',
-            'index_unsigned.jar',
+            b'index_unsigned.jar',
-            'index.xml',
+            b'index.xml',
-            'index.html',
+            b'index.html',
-            'index-v1.jar',
+            b'index-v1.jar',
-            'index-v1.json',
+            b'index-v1.json',
-            'categories.txt',
+            b'categories.txt',
         ]

fdroidserver/index.py

@@ -139,7 +139,7 @@ def make(apps, sortedids, apks, repodir, archive):
                 appsWithPackages[packageName] = newapp
                 break
 
-    requestsdict = dict()
+    requestsdict = collections.OrderedDict()
     for command in ('install', 'uninstall'):
         packageNames = []
         key = command + '_list'
@@ -199,7 +199,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
                 k = k[:1].lower() + k[1:]
             d[k] = v
 
-    output_packages = dict()
+    output_packages = collections.OrderedDict()
     output['packages'] = output_packages
     for package in packages:
         packageName = package['packageName']
@@ -414,7 +414,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
                     if perm_name.startswith("android.permission."):
                         perm_name = perm_name[19:]
                     old_permissions.add(perm_name)
-                addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
+                addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel)
 
                 for permission in sorted_permissions:
                     permel = doc.createElement('uses-permission')
@@ -436,14 +436,14 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
                 and common.config['make_current_version_link'] \
                 and repodir == 'repo':  # only create these
             namefield = common.config['current_version_name_source']
-            sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
+            sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
-            apklinkname = sanitized_name + '.apk'
+            apklinkname = sanitized_name + b'.apk'
-            current_version_path = os.path.join(repodir, current_version_file)
+            current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
             if os.path.islink(apklinkname):
                 os.remove(apklinkname)
             os.symlink(current_version_path, apklinkname)
             # also symlink gpg signature, if it exists
-            for extension in ('.asc', '.sig'):
+            for extension in (b'.asc', b'.sig'):
                 sigfile_path = current_version_path + extension
                 if os.path.exists(sigfile_path):
                     siglinkname = apklinkname + extension

fdroidserver/metadata.py

@@ -815,6 +815,8 @@ def post_metadata_parse(app):
 
     if isinstance(app.Categories, str):
         app.Categories = [app.Categories]
+    elif app.Categories is None:
+        app.Categories = ['None']
     else:
         app.Categories = [str(i) for i in app.Categories]
 

fdroidserver/update.py

@@ -656,13 +656,15 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
 
     cachechanged = False
     repo_files = []
+    repodir = repodir.encode('utf-8')
     for name in os.listdir(repodir):
         file_extension = common.get_file_extension(name)
         if file_extension == 'apk' or file_extension == 'obb':
             continue
         filename = os.path.join(repodir, name)
-        if filename.endswith('_src.tar.gz'):
+        name_utf8 = name.decode('utf-8')
-            logging.debug('skipping source tarball: ' + filename)
+        if filename.endswith(b'_src.tar.gz'):
+            logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
             continue
         if not common.is_repo_file(filename):
             continue
@@ -682,35 +684,35 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
                     repo_file['added'] = a
                 else:
                     repo_file['added'] = datetime(*a[:6])
-            if repo_file['hash'] == shasum:
+            if repo_file.get('hash') == shasum:
-                logging.debug("Reading " + name + " from cache")
+                logging.debug("Reading " + name_utf8 + " from cache")
                 usecache = True
             else:
                 logging.debug("Ignoring stale cache data for " + name)
 
         if not usecache:
-            logging.debug("Processing " + name)
+            logging.debug("Processing " + name_utf8)
-            repo_file = {}
+            repo_file = collections.OrderedDict()
             # TODO rename apkname globally to something more generic
-            repo_file['name'] = name
+            repo_file['name'] = name_utf8
-            repo_file['apkName'] = name
+            repo_file['apkName'] = name_utf8
             repo_file['hash'] = shasum
             repo_file['hashType'] = 'sha256'
             repo_file['versionCode'] = 0
             repo_file['versionName'] = shasum
             # the static ID is the SHA256 unless it is set in the metadata
             repo_file['packageName'] = shasum
-            n = name.split('_')
+            n = name_utf8.split('_')
             if len(n) == 2:
                 packageName = n[0]
                 versionCode = n[1].split('.')[0]
-                if re.match(r'^-?[0-9]+$', versionCode) \
+                if re.match('^-?[0-9]+$', versionCode) \
-                   and common.is_valid_package_name(name.split('_')[0]):
+                   and common.is_valid_package_name(name_utf8.split('_')[0]):
                     repo_file['packageName'] = packageName
                     repo_file['versionCode'] = int(versionCode)
-            srcfilename = name + "_src.tar.gz"
+            srcfilename = name + b'_src.tar.gz'
             if os.path.exists(os.path.join(repodir, srcfilename)):
-                repo_file['srcname'] = srcfilename
+                repo_file['srcname'] = srcfilename.decode('utf-8')
             repo_file['size'] = stat.st_size
 
             apkcache[name] = repo_file
@@ -759,7 +761,7 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
     usecache = False
     if apkfilename in apkcache:
         apk = apkcache[apkfilename]
-        if apk['hash'] == shasum:
+        if apk.get('hash') == shasum:
             logging.debug("Reading " + apkfilename + " from cache")
             usecache = True
         else:
@@ -775,9 +777,9 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
         if os.path.exists(os.path.join(repodir, srcfilename)):
             apk['srcname'] = srcfilename
         apk['size'] = os.path.getsize(apkfile)
-        apk['uses-permission'] = set()
+        apk['uses-permission'] = []
-        apk['uses-permission-sdk-23'] = set()
+        apk['uses-permission-sdk-23'] = []
-        apk['features'] = set()
+        apk['features'] = []
         apk['icons_src'] = {}
         apk['icons'] = {}
         apk['antiFeatures'] = set()
@@ -1065,7 +1067,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
             os.makedirs(icon_dir)
 
     apks = []
-    for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
+    for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))):
         apkfilename = apkfile[len(repodir) + 1:]
         (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk)
         if skip:

tests/index.TestCase

@@ -1,9 +1,18 @@
 #!/usr/bin/env python3
+
+import inspect
 import optparse
 import os
+import sys
 import unittest
 import zipfile
 
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
 import fdroidserver.common
 import fdroidserver.index
 import fdroidserver.signindex