From baeae862ad15df19c430cb4e9d806e94f3b59dec Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 22 Mar 2017 10:59:44 +0100 Subject: [PATCH 1/6] buildserver: allow gradle to install new components Google is making gradle automatically download Android SDK components that are needed by the build, but not already present. We need to support that since it would be a lot of work to fight it. Plus, since each build starts from the fresh snapshot, it should not be such a big deal to let each build install stuff during the process. closes #268 --- buildserver/provision-android-sdk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index e5a4202a..54f89cb6 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -82,6 +82,8 @@ echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;sup echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.2" chmod -R a+rX $ANDROID_HOME/ +chgrp vagrant $ANDROID_HOME +chmod g+w $ANDROID_HOME find $ANDROID_HOME/ -type f -executable -print0 | xargs -0 chmod a+x # allow gradle to install newer build-tools versions From 7d4176dd501edaee974ccc9d75826ab6cf7c6bff Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 23 Mar 2017 23:13:37 +0100 Subject: [PATCH 2/6] metadata: don't crash if metadata includes blank Categories list If someone includes just 'Categories:', it shouldn't crash. --- fdroidserver/metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index da4730cb..1d978dcc 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -815,6 +815,8 @@ def post_metadata_parse(app): if isinstance(app.Categories, str): app.Categories = [app.Categories] + elif app.Categories is None: + app.Categories = ['None'] else: app.Categories = [str(i) for i in app.Categories] From 5ad3486741cee7d76d251cc27a49e1b4fa7310b9 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 11:29:21 +0200 Subject: [PATCH 3/6] make sure indexes are sorted to minimize diffs between copies With the binary transparency log stored in git, it makes that more readable and as small as possible if the index files are fully sorted. That will reduce the differences between two copies of an index file to a bare minimum. If in the future we implement some kind of binary diff transfer, this will also help there. --- fdroidserver/index.py | 6 +++--- fdroidserver/update.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 1421acb8..c284ce50 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -139,7 +139,7 @@ def make(apps, sortedids, apks, repodir, archive): appsWithPackages[packageName] = newapp break - requestsdict = dict() + requestsdict = collections.OrderedDict() for command in ('install', 'uninstall'): packageNames = [] key = command + '_list' @@ -199,7 +199,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict): k = k[:1].lower() + k[1:] d[k] = v - output_packages = dict() + output_packages = collections.OrderedDict() output['packages'] = output_packages for package in packages: packageName = package['packageName'] @@ -414,7 +414,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): if perm_name.startswith("android.permission."): perm_name = perm_name[19:] old_permissions.add(perm_name) - addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel) + addElementNonEmpty('permissions', ','.join(sorted(old_permissions)), doc, apkel) for permission in sorted_permissions: permel = doc.createElement('uses-permission') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 9f5c280c..f8bf731f 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -690,7 +690,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): if not usecache: logging.debug("Processing " + name) - repo_file = {} + repo_file = collections.OrderedDict() # TODO rename apkname globally to something more generic repo_file['name'] = name repo_file['apkName'] = name @@ -775,9 +775,9 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): if os.path.exists(os.path.join(repodir, srcfilename)): apk['srcname'] = srcfilename apk['size'] = os.path.getsize(apkfile) - apk['uses-permission'] = set() - apk['uses-permission-sdk-23'] = set() - apk['features'] = set() + apk['uses-permission'] = [] + apk['uses-permission-sdk-23'] = [] + apk['features'] = [] apk['icons_src'] = {} apk['icons'] = {} apk['antiFeatures'] = set() @@ -1065,7 +1065,7 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): os.makedirs(icon_dir) apks = [] - for apkfile in glob.glob(os.path.join(repodir, '*.apk')): + for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))): apkfilename = apkfile[len(repodir) + 1:] (skip, apk, cachechanged) = scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk) if skip: From 4d50ab9badcd57b769644450d3d6872ebcc329f2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 16:52:17 +0200 Subject: [PATCH 4/6] ignore cache on key name change, instead of crashing In 6c2cf2ccdd7130a034e50f9bea481351d475a272, the names of some essential data bits changed. If those names are not in the tmp/apkcache, then `fdroid update` shouldn't crash but instead just ignore that cache entry. tmp/apkcache should be deleted since the metadata version was bumped, but I guess that does not always happen. --- fdroidserver/update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index f8bf731f..6412029b 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -682,7 +682,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): repo_file['added'] = a else: repo_file['added'] = datetime(*a[:6]) - if repo_file['hash'] == shasum: + if repo_file.get('hash') == shasum: logging.debug("Reading " + name + " from cache") usecache = True else: @@ -759,7 +759,7 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): usecache = False if apkfilename in apkcache: apk = apkcache[apkfilename] - if apk['hash'] == shasum: + if apk.get('hash') == shasum: logging.debug("Reading " + apkfilename + " from cache") usecache = True else: From e58ad330f4ab6900c12f84d0f242de1f0446cab2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 20:24:00 +0200 Subject: [PATCH 5/6] encode filenames as bytes to handle all locale setups This was failing on environments that did not have any LANG or LC_* locale variables set. This is a valid setup, and is common in headless setups, so it needs to be handled. This also adds a new pass of the test suite without the locale env vars set so that this situation is also tests on gitlab-ci, not only gpjenkins. The error this caused was: UnicodeEncodeError: 'ascii' codec can't encode characters in position 6-18: ordinal not in range(128) --- .gitlab-ci.yml | 2 +- fdroidserver/common.py | 23 +++++++++++++---------- fdroidserver/index.py | 8 ++++---- fdroidserver/update.py | 24 +++++++++++++----------- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d1713c0..cbda46b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: fdroid/ci:server-20161223 +image: registry.gitlab.com/fdroid/ci-images:server-latest test: script: diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 42918ed7..0c12c3f9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1700,7 +1700,8 @@ class KnownApks: def get_file_extension(filename): """get the normalized file extension, can be blank string but never None""" - + if isinstance(filename, bytes): + filename = filename.decode('utf-8') return os.path.splitext(filename)[1].lower()[1:] @@ -2333,15 +2334,17 @@ def get_per_app_repos(): def is_repo_file(filename): '''Whether the file in a repo is a build product to be delivered to users''' + if isinstance(filename, str): + filename = filename.encode('utf-8', errors="surrogateescape") return os.path.isfile(filename) \ - and not filename.endswith('.asc') \ - and not filename.endswith('.sig') \ + and not filename.endswith(b'.asc') \ + and not filename.endswith(b'.sig') \ and os.path.basename(filename) not in [ - 'index.jar', - 'index_unsigned.jar', - 'index.xml', - 'index.html', - 'index-v1.jar', - 'index-v1.json', - 'categories.txt', + b'index.jar', + b'index_unsigned.jar', + b'index.xml', + b'index.html', + b'index-v1.jar', + b'index-v1.json', + b'categories.txt', ] diff --git a/fdroidserver/index.py b/fdroidserver/index.py index c284ce50..b42c63d1 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -436,14 +436,14 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): and common.config['make_current_version_link'] \ and repodir == 'repo': # only create these namefield = common.config['current_version_name_source'] - sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield)) - apklinkname = sanitized_name + '.apk' - current_version_path = os.path.join(repodir, current_version_file) + sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8')) + apklinkname = sanitized_name + b'.apk' + current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape') if os.path.islink(apklinkname): os.remove(apklinkname) os.symlink(current_version_path, apklinkname) # also symlink gpg signature, if it exists - for extension in ('.asc', '.sig'): + for extension in (b'.asc', b'.sig'): sigfile_path = current_version_path + extension if os.path.exists(sigfile_path): siglinkname = apklinkname + extension diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 6412029b..0e42d684 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -656,13 +656,15 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): cachechanged = False repo_files = [] + repodir = repodir.encode('utf-8') for name in os.listdir(repodir): file_extension = common.get_file_extension(name) if file_extension == 'apk' or file_extension == 'obb': continue filename = os.path.join(repodir, name) - if filename.endswith('_src.tar.gz'): - logging.debug('skipping source tarball: ' + filename) + name_utf8 = name.decode('utf-8') + if filename.endswith(b'_src.tar.gz'): + logging.debug('skipping source tarball: ' + filename.decode('utf-8')) continue if not common.is_repo_file(filename): continue @@ -683,34 +685,34 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): else: repo_file['added'] = datetime(*a[:6]) if repo_file.get('hash') == shasum: - logging.debug("Reading " + name + " from cache") + logging.debug("Reading " + name_utf8 + " from cache") usecache = True else: logging.debug("Ignoring stale cache data for " + name) if not usecache: - logging.debug("Processing " + name) + logging.debug("Processing " + name_utf8) repo_file = collections.OrderedDict() # TODO rename apkname globally to something more generic - repo_file['name'] = name - repo_file['apkName'] = name + repo_file['name'] = name_utf8 + repo_file['apkName'] = name_utf8 repo_file['hash'] = shasum repo_file['hashType'] = 'sha256' repo_file['versionCode'] = 0 repo_file['versionName'] = shasum # the static ID is the SHA256 unless it is set in the metadata repo_file['packageName'] = shasum - n = name.split('_') + n = name_utf8.split('_') if len(n) == 2: packageName = n[0] versionCode = n[1].split('.')[0] - if re.match(r'^-?[0-9]+$', versionCode) \ - and common.is_valid_package_name(name.split('_')[0]): + if re.match('^-?[0-9]+$', versionCode) \ + and common.is_valid_package_name(name_utf8.split('_')[0]): repo_file['packageName'] = packageName repo_file['versionCode'] = int(versionCode) - srcfilename = name + "_src.tar.gz" + srcfilename = name + b'_src.tar.gz' if os.path.exists(os.path.join(repodir, srcfilename)): - repo_file['srcname'] = srcfilename + repo_file['srcname'] = srcfilename.decode('utf-8') repo_file['size'] = stat.st_size apkcache[name] = repo_file From 53b69f507eac5415d1891612ecbe2aec623a5192 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 3 Apr 2017 19:19:37 +0200 Subject: [PATCH 6/6] allow index.TestCase to work when run using any path --- tests/index.TestCase | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/index.TestCase b/tests/index.TestCase index bbfac7bf..780da775 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -1,9 +1,18 @@ #!/usr/bin/env python3 + +import inspect import optparse import os +import sys import unittest import zipfile +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + import fdroidserver.common import fdroidserver.index import fdroidserver.signindex