diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d1713c0..cbda46b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: fdroid/ci:server-20161223 +image: registry.gitlab.com/fdroid/ci-images:server-latest test: script: diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index e5a4202a..54f89cb6 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -82,6 +82,8 @@ echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;sup echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.2" chmod -R a+rX $ANDROID_HOME/ +chgrp vagrant $ANDROID_HOME +chmod g+w $ANDROID_HOME find $ANDROID_HOME/ -type f -executable -print0 | xargs -0 chmod a+x # allow gradle to install newer build-tools versions diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 42918ed7..0c12c3f9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1700,7 +1700,8 @@ class KnownApks: def get_file_extension(filename): """get the normalized file extension, can be blank string but never None""" - + if isinstance(filename, bytes): + filename = filename.decode('utf-8') return os.path.splitext(filename)[1].lower()[1:] @@ -2333,15 +2334,17 @@ def get_per_app_repos(): def is_repo_file(filename): '''Whether the file in a repo is a build product to be delivered to users''' + if isinstance(filename, str): + filename = filename.encode('utf-8', errors="surrogateescape") return os.path.isfile(filename) \ - and not filename.endswith('.asc') \ - and not filename.endswith('.sig') \ + and not filename.endswith(b'.asc') \ + and not filename.endswith(b'.sig') \ and os.path.basename(filename) not in [ - 'index.jar', - 'index_unsigned.jar', - 'index.xml', - 'index.html', - 'index-v1.jar', - 'index-v1.json', - 'categories.txt', + b'index.jar', + b'index_unsigned.jar', + b'index.xml', + b'index.html', + b'index-v1.jar', + b'index-v1.json', + b'categories.txt', ] diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 1421acb8..b42c63d1 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -139,7 +139,7 @@ def make(apps, sortedids, apks, repodir, archive): appsWithPackages[packageName] = newapp break - requestsdict = dict() + requestsdict = collections.OrderedDict() for command in ('install', 'uninstall'): packageNames = [] key = command + '_list' @@ -199,7 +199,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict): k = k[:1].lower() + k[1:] d[k] = v - output_packages = dict() + output_packages = collections.OrderedDict() output['packages'] = output_packages for package in packages: packageName = package['packageName'] @@ -414,7 +414,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): if perm_name.startswith("android.permission."): perm_name = perm_name[19:] old_permissions.add(perm_name) - addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel) + addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel) for permission in sorted_permissions: permel = doc.createElement('uses-permission') @@ -436,14 +436,14 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): and common.config['make_current_version_link'] \ and repodir == 'repo': # only create these namefield = common.config['current_version_name_source'] - sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield)) - apklinkname = sanitized_name + '.apk' - current_version_path = os.path.join(repodir, current_version_file) + sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8')) + apklinkname = sanitized_name + b'.apk' + current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape') if os.path.islink(apklinkname): os.remove(apklinkname) os.symlink(current_version_path, apklinkname) # also symlink gpg signature, if it exists - for extension in ('.asc', '.sig'): + for extension in (b'.asc', b'.sig'): sigfile_path = current_version_path + extension if os.path.exists(sigfile_path): siglinkname = apklinkname + extension diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index da4730cb..1d978dcc 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -815,6 +815,8 @@ def post_metadata_parse(app): if isinstance(app.Categories, str): app.Categories = [app.Categories] + elif app.Categories is None: + app.Categories = ['None'] else: app.Categories = [str(i) for i in app.Categories] diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 9f5c280c..0e42d684 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -656,13 +656,15 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): cachechanged = False repo_files = [] + repodir = repodir.encode('utf-8') for name in os.listdir(repodir): file_extension = common.get_file_extension(name) if file_extension == 'apk' or file_extension == 'obb': continue filename = os.path.join(repodir, name) - if filename.endswith('_src.tar.gz'): - logging.debug('skipping source tarball: ' + filename) + name_utf8 = name.decode('utf-8') + if filename.endswith(b'_src.tar.gz'): + logging.debug('skipping source tarball: ' + filename.decode('utf-8')) continue if not common.is_repo_file(filename): continue @@ -682,35 +684,35 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_file['added'] = a else: repo_file['added'] = datetime(*a[:6]) - if repo_file['hash'] == shasum: - logging.debug("Reading " + name + " from cache") + if repo_file.get('hash') == shasum: + logging.debug("Reading " + name_utf8 + " from cache") usecache = True else: logging.debug("Ignoring stale cache data for " + name) if not usecache: - logging.debug("Processing " + name) - repo_file = {} + logging.debug("Processing " + name_utf8) + repo_file = collections.OrderedDict() # TODO rename apkname globally to something more generic - repo_file['name'] = name - repo_file['apkName'] = name + repo_file['name'] = name_utf8 + repo_file['apkName'] = name_utf8 repo_file['hash'] = shasum repo_file['hashType'] = 'sha256' repo_file['versionCode'] = 0 repo_file['versionName'] = shasum # the static ID is the SHA256 unless it is set in the metadata repo_file['packageName'] = shasum - n = name.split('_') + n = name_utf8.split('_') if len(n) == 2: packageName = n[0] versionCode = n[1].split('.')[0] - if re.match(r'^-?[0-9]+$', versionCode) \ - and common.is_valid_package_name(name.split('_')[0]): + if re.match('^-?[0-9]+$', versionCode) \ + and common.is_valid_package_name(name_utf8.split('_')[0]): repo_file['packageName'] = packageName repo_file['versionCode'] = int(versionCode) - srcfilename = name + "_src.tar.gz" + srcfilename = name + b'_src.tar.gz' if os.path.exists(os.path.join(repodir, srcfilename)): - repo_file['srcname'] = srcfilename + repo_file['srcname'] = srcfilename.decode('utf-8') repo_file['size'] = stat.st_size apkcache[name] = repo_file @@ -759,7 +761,7 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): usecache = False if apkfilename in apkcache: apk = apkcache[apkfilename] - if apk['hash'] == shasum: + if apk.get('hash') == shasum: logging.debug("Reading " + apkfilename + " from cache") usecache = True else: @@ -775,9 +777,9 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): if os.path.exists(os.path.join(repodir, srcfilename)): apk['srcname'] = srcfilename apk['size'] = os.path.getsize(apkfile) - apk['uses-permission'] = set() - apk['uses-permission-sdk-23'] = set() - apk['features'] = set() + apk['uses-permission'] = [] + apk['uses-permission-sdk-23'] = [] + apk['features'] = [] apk['icons_src'] = {} apk['icons'] = {} apk['antiFeatures'] = set() @@ -1065,7 +1067,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): os.makedirs(icon_dir) apks = [] - for apkfile in glob.glob(os.path.join(repodir, '*.apk')): + for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))): apkfilename = apkfile[len(repodir) + 1:] (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk) if skip: diff --git a/tests/index.TestCase b/tests/index.TestCase index bbfac7bf..780da775 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -1,9 +1,18 @@ #!/usr/bin/env python3 + +import inspect import optparse import os +import sys import unittest import zipfile +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + import fdroidserver.common import fdroidserver.index import fdroidserver.signindex