From 9f553186e8d24559ab8a9ce2745c041fc094dac4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 16 Feb 2018 14:25:47 +0100 Subject: [PATCH 01/12] gitlab-ci: switch debian_testing to use androguard --- .gitlab-ci.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1705910f..c315e1ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,15 +46,13 @@ debian_testing: - apt update -y - apt dist-upgrade -y - apt-get install -y --no-install-recommends - aapt adb android-platform-tools-base android-sdk-common fdroidserver + aapt adb androguard android-platform-tools-base android-sdk-common fdroidserver git gnupg python3-setuptools zipalign + - python3 -c 'import fdroidserver' + - python3 -c 'import androguard' - export ANDROID_HOME=/usr/lib/android-sdk - export LANG=C.UTF-8 - cd tests - - rm -f install.TestCase # fails frequently and is unimportant - - echo "Debian's build-tools is too old, remove once the package has been updated" - - sed -i '/android.permission.READ_EXTERNAL_STORAGE/d' repo/index.xml - - sed -i '/^diff -uw .*index-v1.json$/d' run-tests - ./run-tests ubuntu_lts: From 63d4d462914daae01dbccfff6219f6ccad7f4cf2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 20 Feb 2018 17:08:55 +0100 Subject: [PATCH 02/12] update: 'features' list only includes required features The F-Droid index 'features' list is not the same as what is in the AndroidManifest.xml. It only includes "required" features, for example. --- fdroidserver/update.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 51112982..1da8405f 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1268,7 +1268,9 @@ def scan_apk_androguard(apk, apkfile): and feature != "android.hardware.screen.landscape": if feature.startswith("android.feature."): feature = feature[16:] - apk['features'].append(feature) + required = item.attrib.get('{' + xml.nsmap['android'] + '}required') + if required is None or required == 'true': + apk['features'].append(feature) def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False, From aa4f54bf187e2ca45838e82fc50d9a92e4f08ce2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 21 Feb 2018 12:34:54 +0100 Subject: [PATCH 03/12] update: include implied permissions when using androguard `aapt dump badging` includes these when listing uses-permissions: https://github.com/androguard/androguard/pull/428 --- fdroidserver/update.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 1da8405f..7d6f4541 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1251,6 +1251,12 @@ def scan_apk_androguard(apk, apkfile): maxSdkVersion ) apk['uses-permission'].append(permission) + for name, maxSdkVersion in apkobject.get_uses_implied_permission_list(): + permission = UsesPermission( + name, + maxSdkVersion + ) + apk['uses-permission'].append(permission) for item in xml.findall('uses-permission-sdk-23'): name = str(item.attrib['{' + xml.nsmap['android'] + '}name']) From 847bbb6e43805513e6df77241b6c00fb2715f222 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Feb 2018 12:10:53 +0100 Subject: [PATCH 04/12] init: do not try to find aapt if androguard is available --- fdroidserver/init.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 7b59bdb7..b47d65b6 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -131,7 +131,9 @@ def main(): logging.info('Try running `fdroid init` in an empty directory.') raise FDroidException('Repository already exists.') - if 'aapt' not in test_config or not os.path.isfile(test_config['aapt']): + if common.use_androguard(): + pass + elif 'aapt' not in test_config or not os.path.isfile(test_config['aapt']): # try to find a working aapt, in all the recent possible paths build_tools = os.path.join(test_config['sdk_path'], 'build-tools') aaptdirs = [] From 52b3436ff6013244982709ac3400bcb60a45f31a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Feb 2018 15:08:53 +0100 Subject: [PATCH 05/12] make is_apk_and_debuggable() default to using androguard before aapt --- fdroidserver/build.py | 2 +- fdroidserver/common.py | 30 ++++++++++++++++++++++++------ fdroidserver/update.py | 2 +- tests/common.TestCase | 4 ++-- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 41fb2251..58e4ccf4 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -817,7 +817,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext src = os.path.normpath(apks[0]) # Make sure it's not debuggable... - if common.isApkAndDebuggable(src): + if common.is_apk_and_debuggable(src): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 87a9d279..9c20443a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1927,7 +1927,25 @@ def get_file_extension(filename): return os.path.splitext(filename)[1].lower()[1:] -def get_apk_debuggable_aapt(apkfile): +def use_androguard(): + """Report if androguard is available, and config its debug logging""" + + try: + import androguard + if use_androguard.show_path: + logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__)) + use_androguard.show_path = False + if options and options.verbose: + logging.getLogger("androguard.axml").setLevel(logging.INFO) + return True + except ImportError: + return False + + +use_androguard.show_path = True + + +def is_apk_and_debuggable_aapt(apkfile): p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'], output=False) if p.returncode != 0: @@ -1938,7 +1956,7 @@ def get_apk_debuggable_aapt(apkfile): return False -def get_apk_debuggable_androguard(apkfile): +def is_apk_and_debuggable_androguard(apkfile): try: from androguard.core.bytecodes.apk import APK except ImportError: @@ -1952,7 +1970,7 @@ def get_apk_debuggable_androguard(apkfile): return False -def isApkAndDebuggable(apkfile): +def is_apk_and_debuggable(apkfile): """Returns True if the given file is an APK and is debuggable :param apkfile: full path to the apk to check""" @@ -1960,10 +1978,10 @@ def isApkAndDebuggable(apkfile): if get_file_extension(apkfile) != 'apk': return False - if SdkToolsPopen(['aapt', 'version'], output=False): - return get_apk_debuggable_aapt(apkfile) + if use_androguard(): + return is_apk_and_debuggable_androguard(apkfile) else: - return get_apk_debuggable_androguard(apkfile) + return is_apk_and_debuggable_aapt(apkfile) def get_apk_id_aapt(apkfile): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 7d6f4541..5b39e751 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1324,7 +1324,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal return True, None, False # Check for debuggable apks... - if common.isApkAndDebuggable(apkfile): + if common.is_apk_and_debuggable(apkfile): logging.warning('{0} is set to android:debuggable="true"'.format(apkfile)) if options.rename_apks: diff --git a/tests/common.TestCase b/tests/common.TestCase index 330d37a7..1d5678dd 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -140,7 +140,7 @@ class CommonTest(unittest.TestCase): testfiles.append(os.path.join(self.basedir, 'urzip-badsig.apk')) testfiles.append(os.path.join(self.basedir, 'urzip-badcert.apk')) for apkfile in testfiles: - debuggable = fdroidserver.common.isApkAndDebuggable(apkfile) + debuggable = fdroidserver.common.is_apk_and_debuggable(apkfile) self.assertTrue(debuggable, "debuggable APK state was not properly parsed!") # these are set NOT debuggable @@ -148,7 +148,7 @@ class CommonTest(unittest.TestCase): testfiles.append(os.path.join(self.basedir, 'urzip-release.apk')) testfiles.append(os.path.join(self.basedir, 'urzip-release-unsigned.apk')) for apkfile in testfiles: - debuggable = fdroidserver.common.isApkAndDebuggable(apkfile) + debuggable = fdroidserver.common.is_apk_and_debuggable(apkfile) self.assertFalse(debuggable, "debuggable APK state was not properly parsed!") From 11bed21071b8b615234f3481cd32861677f3bd08 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Feb 2018 14:40:03 +0100 Subject: [PATCH 06/12] gitlab-ci: androguard from Debian/unstable til it stabilizes --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c315e1ee..3fe13512 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,8 +46,10 @@ debian_testing: - apt update -y - apt dist-upgrade -y - apt-get install -y --no-install-recommends - aapt adb androguard android-platform-tools-base android-sdk-common fdroidserver - git gnupg python3-setuptools zipalign + fdroidserver git gnupg python3-setuptools + - sed -i -e 's,testing,sid,g' -e 's,testing,sid,g' /etc/apt/sources.list + - apt-get update -y + - apt-get install -y --no-install-recommends aapt androguard android-platform-tools-base zipalign - python3 -c 'import fdroidserver' - python3 -c 'import androguard' - export ANDROID_HOME=/usr/lib/android-sdk From 89498208fc95b6c3064ccb3724e69e821f42a733 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 21 Feb 2018 13:59:34 +0100 Subject: [PATCH 07/12] gitlab-ci: test against latest build-tools 27.0.3 --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fe13512..74e33abd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,10 +72,10 @@ ubuntu_lts: git gnupg python3-setuptools unzip wget zipalign - export ANDROID_HOME=/usr/lib/android-sdk # xenial's aapt is too old - - wget --no-verbose https://dl.google.com/android/repository/build-tools_r27.0.1-linux.zip - - unzip -q build-tools_r27.0.1-linux.zip - - rm build-tools_r27.0.1-linux.zip - - mv android-8.1.0 $ANDROID_HOME/build-tools/27.0.1 + - wget --no-verbose https://dl.google.com/android/repository/build-tools_r27.0.3-linux.zip + - unzip -q build-tools_r27.0.3-linux.zip + - rm build-tools_r27.0.3-linux.zip + - mv android-8.1.0 $ANDROID_HOME/build-tools/27.0.3 - export LANG=C.UTF-8 - cd tests - ./run-tests From 089712c01292084795014187f4afeb4c49306242 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 21 Feb 2018 14:08:03 +0100 Subject: [PATCH 08/12] tests: do not automatically run install.TestCase, its troublesome `fdroid install` is rarely used, if at all, and the test frequently fails for no reason in gitlab-ci, because it can't start the adb server. --- tests/run-tests | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/run-tests b/tests/run-tests index 13a96a37..81f0afc9 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -125,7 +125,11 @@ cd $WORKSPACE/tests/getsig ./make.sh cd $WORKSPACE/tests -for testcase in $WORKSPACE/tests/*.TestCase; do +for testcase in $WORKSPACE/tests/i*.TestCase; do + if [ $testcase == $WORKSPACE/tests/install.TestCase ]; then + echo "skipping install.TestCase, its too troublesome in CI builds" + continue + fi $testcase done From 0e9252db371eb53c844cfa8440f6e1c9a4754cf0 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Feb 2018 10:27:10 +0100 Subject: [PATCH 09/12] gitlab-ci: try to download PPA keys until they succeed This download occasionally fails, so this keeps retrying till it succeeds. The CI job has a time limit, so no need to figure out an exit condition. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74e33abd..18e282da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,7 @@ ubuntu_lts: only: - master@fdroid/fdroidserver script: - - apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151 + - while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1` - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list - apt update -y From f420a037d55e4b78bb382040147d97ba37ac1c44 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Feb 2018 10:32:27 +0100 Subject: [PATCH 10/12] gitlab-ci: remove apt-get progress dumps from build log --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18e282da..9a6cc9b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,12 +43,12 @@ debian_testing: only: - master@fdroid/fdroidserver script: - - apt update -y - - apt dist-upgrade -y - - apt-get install -y --no-install-recommends + - apt-get -qy update + - apt-get -qy dist-upgrade + - apt-get -qy install --no-install-recommends fdroidserver git gnupg python3-setuptools - sed -i -e 's,testing,sid,g' -e 's,testing,sid,g' /etc/apt/sources.list - - apt-get update -y + - apt-get -qy update - apt-get install -y --no-install-recommends aapt androguard android-platform-tools-base zipalign - python3 -c 'import fdroidserver' - python3 -c 'import androguard' @@ -65,9 +65,9 @@ ubuntu_lts: - while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1` - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list - - apt update -y - - apt dist-upgrade -y - - apt-get install -y --no-install-recommends + - apt-get -qy update + - apt-get -qy dist-upgrade + - apt-get -qy install --no-install-recommends aapt adb android-platform-tools-base android-sdk-common fdroidserver git gnupg python3-setuptools unzip wget zipalign - export ANDROID_HOME=/usr/lib/android-sdk From 88e24dc4e305e1e7476dfb23001bc40387f0de10 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Feb 2018 13:39:41 +0100 Subject: [PATCH 11/12] update: switch to improved androguard detection --- fdroidserver/update.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 5b39e751..4d227dd4 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1049,11 +1049,9 @@ def scan_apk(apk_file): 'antiFeatures': set(), } - try: - import androguard - androguard # silence pyflakes + if common.use_androguard(): scan_apk_androguard(apk, apk_file) - except ImportError: + else: scan_apk_aapt(apk, apk_file) # Get the signature, or rather the signing key fingerprints From e75bf70be6b89eedabbb2ea5b33de9e92aba3f2b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 22 Feb 2018 14:30:39 +0100 Subject: [PATCH 12/12] signatures: future-proof fetching app ID info from APK We're not using platformBuildVersionName and it might go away just like it appeared: with no good reason or announcement. --- fdroidserver/common.py | 2 +- tests/common.TestCase | 38 ++++++++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 9c20443a..b5ff274c 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1990,7 +1990,7 @@ def get_apk_id_aapt(apkfile): :param apkfile: path to an APK file. :returns: triplet (appid, version code, version name) """ - r = re.compile("package: name='(?P.*)' versionCode='(?P.*)' versionName='(?P.*)' platformBuildVersionName='.*'") + r = re.compile("^package: name='(?P.*)' versionCode='(?P.*)' versionName='(?P.*)'.*") p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) for line in p.output.splitlines(): m = r.match(line) diff --git a/tests/common.TestCase b/tests/common.TestCase index 1d5678dd..55af2558 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -541,10 +541,40 @@ class CommonTest(unittest.TestCase): self._set_build_tools() config['aapt'] = fdroidserver.common.find_sdk_tools_cmd('aapt') - appid, vercode, vername = fdroidserver.common.get_apk_id_aapt('repo/obb.main.twoversions_1101613.apk') - self.assertEqual('obb.main.twoversions', appid) - self.assertEqual('1101613', vercode) - self.assertEqual('0.1', vername) + testcases = [ + ('repo/obb.main.twoversions_1101613.apk', 'obb.main.twoversions', '1101613', '0.1'), + ('OBBMainPatchCurrent.apk', 'obb.mainpatch.current', '1619', '0.1'), + ('OBBMainTwoVersions.apk', 'obb.main.twoversions', '1101613', '0.1'), + ('org.bitbucket.tickytacky.mirrormirror_1.apk', 'org.bitbucket.tickytacky.mirrormirror', '1', '1.0'), + ('org.bitbucket.tickytacky.mirrormirror_2.apk', 'org.bitbucket.tickytacky.mirrormirror', '2', '1.0.1'), + ('org.bitbucket.tickytacky.mirrormirror_3.apk', 'org.bitbucket.tickytacky.mirrormirror', '3', '1.0.2'), + ('org.bitbucket.tickytacky.mirrormirror_4.apk', 'org.bitbucket.tickytacky.mirrormirror', '4', '1.0.3'), + ('org.dyndns.fules.ck_20.apk', 'org.dyndns.fules.ck', '20', 'v1.6pre2'), + ('urzip.apk', 'info.guardianproject.urzip', '100', '0.1'), + ('urzip-badcert.apk', 'info.guardianproject.urzip', '100', '0.1'), + ('urzip-badsig.apk', 'info.guardianproject.urzip', '100', '0.1'), + ('urzip-release.apk', 'info.guardianproject.urzip', '100', '0.1'), + ('urzip-release-unsigned.apk', 'info.guardianproject.urzip', '100', '0.1'), + ('urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk', 'info.guardianproject.urzip', '100', '0.1'), + ('repo/com.politedroid_3.apk', 'com.politedroid', '3', '1.2'), + ('repo/com.politedroid_4.apk', 'com.politedroid', '4', '1.3'), + ('repo/com.politedroid_5.apk', 'com.politedroid', '5', '1.4'), + ('repo/com.politedroid_6.apk', 'com.politedroid', '6', '1.5'), + ('repo/duplicate.permisssions_9999999.apk', 'duplicate.permisssions', '9999999', '0.3-7-gb817ac8'), + ('repo/info.zwanenburg.caffeinetile_4.apk', 'info.zwanenburg.caffeinetile', '4', '1.3'), + ('repo/obb.main.oldversion_1444412523.apk', 'obb.main.oldversion', '1444412523', '0.1'), + ('repo/obb.mainpatch.current_1619_another-release-key.apk', 'obb.mainpatch.current', '1619', '0.1'), + ('repo/obb.mainpatch.current_1619.apk', 'obb.mainpatch.current', '1619', '0.1'), + ('repo/obb.main.twoversions_1101613.apk', 'obb.main.twoversions', '1101613', '0.1'), + ('repo/obb.main.twoversions_1101615.apk', 'obb.main.twoversions', '1101615', '0.1'), + ('repo/obb.main.twoversions_1101617.apk', 'obb.main.twoversions', '1101617', '0.1'), + ('repo/urzip-; Рахма́нинов, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢尔盖·.apk', 'info.guardianproject.urzip', '100', '0.1'), + ] + for apkfilename, appid, versionCode, versionName in testcases: + a, vc, vn = fdroidserver.common.get_apk_id_aapt(apkfilename) + self.assertEqual(appid, a) + self.assertEqual(versionCode, vc) + self.assertEqual(versionName, vn) with self.assertRaises(FDroidException): fdroidserver.common.get_apk_id_aapt('nope')