From 6ca060e10d6ef6df4408dc71ac42c63e3144a40b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 29 May 2014 21:43:16 -0400 Subject: [PATCH 01/15] add more SDK checks: build-tools/19.0.3 and presense of aapt Make sure that fdroid can find aapt in the current config, otherwise exit with an error. Some users don't have build_tools set, and their SDK does not include the build-tools in the default versioned dir, so this should warn them of what is wrong. --- fdroidserver/common.py | 17 +++++++++++++++++ fdroidserver/init.py | 21 ++++++++++++--------- tests/run-tests | 43 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 9854cd32..a5e9bd09 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -144,6 +144,23 @@ def test_sdk_exists(c): return True +def test_build_tools_exists(c): + if not test_sdk_exists(c): + return False + build_tools = os.path.join(c['sdk_path'], 'build-tools') + versioned_build_tools = os.path.join(build_tools, c['build_tools']) + if not os.path.isdir(versioned_build_tools): + logging.critical('Android Build Tools path "' + + versioned_build_tools + '" does not exist!') + return False + if not os.path.exists(os.path.join(c['sdk_path'], 'build-tools', c['build_tools'], 'aapt')): + logging.critical('Android Build Tools "' + + versioned_build_tools + + '" does not contain "aapt"!') + return False + return True + + def write_password_file(pwtype, password=None): ''' writes out passwords to a protected file instead of passing passwords as diff --git a/fdroidserver/init.py b/fdroidserver/init.py index ace07bf8..c51b6e59 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -155,18 +155,15 @@ def main(): logging.info('Try running `fdroid init` in an empty directory.') sys.exit() - # now that we have a local config.py, read configuration... - config = common.read_config(options) - # try to find a working aapt, in all the recent possible paths - build_tools = os.path.join(config['sdk_path'], 'build-tools') + build_tools = os.path.join(test_config['sdk_path'], 'build-tools') aaptdirs = [] - aaptdirs.append(os.path.join(build_tools, config['build_tools'])) + aaptdirs.append(os.path.join(build_tools, test_config['build_tools'])) aaptdirs.append(build_tools) - for f in sorted(os.listdir(build_tools), reverse=True): + for f in os.listdir(build_tools): if os.path.isdir(os.path.join(build_tools, f)): aaptdirs.append(os.path.join(build_tools, f)) - for d in aaptdirs: + for d in sorted(aaptdirs, reverse=True): if os.path.isfile(os.path.join(d, 'aapt')): aapt = os.path.join(d, 'aapt') break @@ -174,9 +171,15 @@ def main(): dirname = os.path.basename(os.path.dirname(aapt)) if dirname == 'build-tools': # this is the old layout, before versioned build-tools - write_to_config('build_tools', '') + test_config['build_tools'] = '' else: - write_to_config('build_tools', dirname) + test_config['build_tools'] = dirname + write_to_config('build_tools', test_config['build_tools']) + if not common.test_build_tools_exists(test_config): + sys.exit(3) + + # now that we have a local config.py, read configuration... + config = common.read_config(options) # track down where the Android NDK is ndk_path = '/opt/android-ndk' diff --git a/tests/run-tests b/tests/run-tests index 2d0ad5cd..f3e9a3e7 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -3,6 +3,11 @@ set -e set -x +echo_header() { + echo "==============================================================================" + echo $1 +} + copy_apks_into_repo() { for f in `ls -1 ../../*/bin/*.apk`; do name=$(basename $(dirname `dirname $f`)) @@ -40,7 +45,7 @@ if [ -z $fdroid ]; then fi #------------------------------------------------------------------------------# -echo "setup a new repo from scratch using ANDROID_HOME" +echo_header "setup a new repo from scratch using ANDROID_HOME" REPOROOT=`create_test_dir` cd $REPOROOT @@ -76,7 +81,33 @@ set -e #------------------------------------------------------------------------------# -echo "check that --android-home overrides ANDROID_HOME" +echo_header "check that fake android home passes `fdroid init`" + +REPOROOT=`create_test_dir` +FAKE_ANDROID_HOME=`create_test_dir` +create_fake_android_home $FAKE_ANDROID_HOME +KEYSTORE=$REPOROOT/keystore.jks +cd $REPOROOT +$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME + + +#------------------------------------------------------------------------------# +echo_header "check that 'fdroid init' fails when build-tools cannot be found" + +REPOROOT=`create_test_dir` +FAKE_ANDROID_HOME=`create_test_dir` +create_fake_android_home $FAKE_ANDROID_HOME +rm -f $FAKE_ANDROID_HOME/build-tools/*/aapt +KEYSTORE=$REPOROOT/keystore.jks +cd $REPOROOT +set +e +$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME +[ $? -eq 0 ] && exit 1 +set -e + + +#------------------------------------------------------------------------------# +echo_header "check that --android-home overrides ANDROID_HOME" REPOROOT=`create_test_dir` FAKE_ANDROID_HOME=`create_test_dir` @@ -94,7 +125,7 @@ set -e #------------------------------------------------------------------------------# -echo "setup a new repo from scratch with keystore and android-home set on cmd line" +echo_header "setup a new repo from scratch with keystore and android-home set on cmd line" REPOROOT=`create_test_dir` KEYSTORE=$REPOROOT/keystore.jks @@ -115,7 +146,7 @@ export ANDROID_HOME=$STORED_ANDROID_HOME #------------------------------------------------------------------------------# -echo "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first" +echo_header "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first" REPOROOT=`create_test_dir` cd $REPOROOT @@ -127,7 +158,7 @@ $fdroid update #------------------------------------------------------------------------------# -echo "setup a new repo from scratch and generate a keystore" +echo_header "setup a new repo from scratch and generate a keystore" REPOROOT=`create_test_dir` KEYSTORE=$REPOROOT/keystore.jks @@ -142,7 +173,7 @@ test -e repo/index.jar #------------------------------------------------------------------------------# -echo "setup a new repo from scratch with a HSM/smartcard" +echo_header "setup a new repo from scratch with a HSM/smartcard" REPOROOT=`create_test_dir` cd $REPOROOT From bc8ad3724986b0282a58d0391666fc90adc776b3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 30 May 2014 11:41:02 -0400 Subject: [PATCH 02/15] init: make prompt for SDK path exit cleanly on Ctrl-C This just makes a clean exit rather than showing a python stacktrace. --- fdroidserver/init.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index c51b6e59..f55549ab 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -132,7 +132,12 @@ def main(): # if neither --android-home nor the default sdk_path exist, prompt the user default_sdk_path = '/opt/android-sdk' while not options.no_prompt: - s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + ') here:\n> ') + try: + s = raw_input('Enter the path to the Android SDK (' + + default_sdk_path + ') here:\n> ') + except KeyboardInterrupt: + print('') + sys.exit(1) if re.match('^\s*$', s) is not None: test_config['sdk_path'] = default_sdk_path else: From ed7a0576c2109fd8f561d2afaf8c0b247c0b1d56 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 30 May 2014 15:53:11 -0400 Subject: [PATCH 03/15] test: make sure `fdroid update -c` creates a full index.xml Previously, `fdroid update -c` would only create the new metadata, but would not add the new apps/apks to the repo. That required a second run of `fdroid update`. This has been fixes, so this test makes sure it stays fixed, in a very generic way. --- tests/run-tests | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index f3e9a3e7..acf31774 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -51,8 +51,8 @@ REPOROOT=`create_test_dir` cd $REPOROOT $fdroid init copy_apks_into_repo $REPOROOT -$fdroid update -c -$fdroid update +$fdroid update --createmeta +grep -F ' repo/$apk" + ln $f $1/repo/$apk || \ + rsync -axv $f $1/repo/$apk # rsync if hard link is not possible + fi done } @@ -34,6 +37,16 @@ create_test_file() { mktemp --tmpdir=$WORKSPACE/.testfiles } +#------------------------------------------------------------------------------# +# "main" + +if [ $# -ne 1 ]; then + echo "Usage: $0 '/path/to/folder/with/apks'" + exit 1 +fi + +APKDIR=$1 + if [ -z $WORKSPACE ]; then WORKSPACE=`dirname $(pwd)` echo "Setting Workspace to $WORKSPACE" From 6085556a7b093d5dd7dfdcfb35a72296e264bb9d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 2 Jun 2014 13:24:24 -0400 Subject: [PATCH 07/15] jenkins: run git pre-commit hook for pep8, pyflakes, etc This provides a central script for running certain checks at commit time, and in jenkins automated builds. --- jenkins-build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-build b/jenkins-build index cbdf115f..35add775 100755 --- a/jenkins-build +++ b/jenkins-build @@ -71,8 +71,8 @@ fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests ~jenkins/ #------------------------------------------------------------------------------# -# run pyflakes -pyflakes fdroid makebuildserver fdroidserver/*.py setup.py +# run git pre-commit hook for pep8, pyflakes, etc +sh hooks/pre-commit #------------------------------------------------------------------------------# From c6d946ed54604ee75131d8dc835834f982b99766 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 2 Jun 2014 13:46:00 -0400 Subject: [PATCH 08/15] update: remove overly verbose message about icon sizing being correct This seems like purely a debug message, I can't see why its useful in general --- fdroidserver/update.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index faf18ba6..0f7d7a9d 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -301,9 +301,6 @@ def resize_icon(iconpath, density): iconpath, oldsize, im.size)) im.save(iconpath, "PNG") - else: - logging.info("%s is small enough: %s" % im.size) - except Exception, e: logging.error("Failed resizing {0} - {1}".format(iconpath, e)) From ed73db65ae13699a405d7c5f4888afb092a4df21 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 2 Jun 2014 16:12:50 -0400 Subject: [PATCH 09/15] make FDroidPopen output based on --verbose FDroidPopen outputs by default, this should be controlled by the --verbose flag so that most of the time, only meaningful messages are shown like errors and such. For command output that should be shown everytime, output=True can be set. --- fdroidserver/common.py | 4 ++-- fdroidserver/update.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a5e9bd09..81675657 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1531,7 +1531,7 @@ def SilentPopen(commands, cwd=None, shell=False): return FDroidPopen(commands, cwd=cwd, shell=shell, output=False) -def FDroidPopen(commands, cwd=None, shell=False, output=True): +def FDroidPopen(commands, cwd=None, shell=False, output=False): """ Run a command and capture the possibly huge output. @@ -1558,7 +1558,7 @@ def FDroidPopen(commands, cwd=None, shell=False, output=True): while not stdout_reader.eof(): while not stdout_queue.empty(): line = stdout_queue.get() - if output and options.verbose: + if output or options.verbose: # Output directly to console sys.stdout.write(line) sys.stdout.flush() diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0f7d7a9d..d7e93fcf 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -363,8 +363,8 @@ def scan_apks(apps, apkcache, repodir, knownapks): thisinfo = apkcache[apkfilename] else: - - logging.info("Processing " + apkfilename) + if options.verbose: + logging.info("Processing " + apkfilename) thisinfo = {} thisinfo['apkname'] = apkfilename srcfilename = apkfilename[:-4] + "_src.tar.gz" @@ -801,8 +801,8 @@ def make_index(apps, apks, repodir, archive, categories): if 'repo_keyalias' in config: - logging.info("Creating signed index with this key:") - logging.info("SHA256: %s" % repo_pubkey_fingerprint) + logging.info("Creating signed index with this key (SHA256):") + logging.info("%s" % repo_pubkey_fingerprint) # Create a jar of the index... p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir) From 57a4e03277c5db92d3f7404f61c54e67ab2af196 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 2 Jun 2014 16:18:12 -0400 Subject: [PATCH 10/15] update: skip APKs that cannot be parsed instead of exiting Some APKs can be corrupt or some system APKs do not have all the normal info. Instead of quitting, `fdroid update` skips the non-parsable APKs and optionally deletes them if --delete-unknown is specified. --- fdroidserver/update.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index d7e93fcf..839a84f8 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -379,8 +379,15 @@ def scan_apks(apps, apkcache, repodir, knownapks): config['build_tools'], 'aapt'), 'dump', 'badging', apkfile]) if p.returncode != 0: - logging.critical("Failed to get apk information") - sys.exit(1) + if options.delete_unknown: + if os.path.exists(apkfile): + logging.error("Failed to get apk information, deleting " + apkfile) + os.remove(apkfile) + else: + logging.error("Could not find {0} to remove it".format(apkfile)) + else: + logging.error("Failed to get apk information, skipping " + apkfile) + continue for line in p.stdout.splitlines(): if line.startswith("package:"): try: From d68830418b5fc755e0a88a9ba26a968d6ba5bf56 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Jun 2014 13:28:35 -0400 Subject: [PATCH 11/15] apps with no get strange values of minSdkVersion Using this example app which does not have : https://android.googlesource.com/platform/development/+log/master/samples/ApiDemos/assets/HelloActivity.apk aapt then returns "sdkVersion:'IceCreamSandwich'". minSdkVersion is only ever supposed to be an integer, so this is a bizarre APK. It is included only as a binary in the git repo for Android sample code. But who knows what else is out there, so report and error and carry on with the update process. --- fdroidserver/update.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 839a84f8..aca6b994 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -419,7 +419,12 @@ def scan_apks(apps, apkcache, repodir, knownapks): path = match.group(2) thisinfo['icons_src'][density] = path elif line.startswith("sdkVersion:"): - thisinfo['sdkversion'] = re.match(sdkversion_pat, line).group(1) + m = re.match(sdkversion_pat, line) + if m is None: + logging.error(line.replace('sdkVersion:', '') + + ' is not a valid minSdkVersion!') + else: + thisinfo['sdkversion'] = m.group(1) elif line.startswith("maxSdkVersion:"): thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1) elif line.startswith("native-code:"): From c81ff764a3286bb72b0d547e67ca044b97ed2d2e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Jun 2014 13:39:11 -0400 Subject: [PATCH 12/15] update: improve warning about APKs set to be debuggable This uses the actual XML code used to set this flag making the warning message easily searchable on the interwebs. --- fdroidserver/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index aca6b994..8cd6b0e3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -452,7 +452,7 @@ def scan_apks(apps, apkcache, repodir, knownapks): # Check for debuggable apks... if common.isApkDebuggable(apkfile, config): - logging.warn("{0} is debuggable... {1}".format(apkfile, line)) + logging.warn('{0} is set to android:debuggable="true"!'.format(apkfile)) # Calculate the sha256... sha = hashlib.sha256() From 13baea955872be1f4c58735eb7747e1fcc47c14c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 3 Jun 2014 14:18:08 -0400 Subject: [PATCH 13/15] some APKs do not have a name, like system APKs It is not necessarily a good idea to try to distribute system APKs via FDroid, but `fdroid update` should just ignore APKs it cannot handle rather than die and prevent a repo from being fully created. This is necessary to handle the automatic creation of repos, like for debug builds from a Jenkins server. --- fdroidserver/update.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 8cd6b0e3..27431c39 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -967,6 +967,9 @@ def main(): break if not found: if options.create_metadata: + if 'name' not in apk: + logging.error(apk['id'] + ' does not have a name! Skipping...') + continue f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w') f.write("License:Unknown\n") f.write("Web Site:\n") From 479e194976bc5cebc9f81c13233d498b657bdf0d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 4 Jun 2014 15:47:54 -0400 Subject: [PATCH 14/15] update name/description in examples/config.py It was confusing that by default, repos created with `fdroid init` had the same name/description as f-droid.org/repo --- examples/config.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/config.py b/examples/config.py index 3f93235f..501a4f1e 100644 --- a/examples/config.py +++ b/examples/config.py @@ -30,14 +30,14 @@ gradle = "gradle" # The same policy is applied to the archive repo, if there is one. repo_maxage = 0 -repo_url = "https://f-droid.org/repo" -repo_name = "F-Droid" +repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo" +repo_name = "My First FDroid Repo Demo" repo_icon = "fdroid-icon.png" repo_description = """ -The official repository of the F-Droid client. Applications in this repository -are either official binaries built by the original application developers, or -are binaries built from source by the admin of f-droid.org using the tools on -https://gitorious.org/f-droid. +This is a repository of apps to be used with FDroid. Applications in this +repository are either official binaries built by the original application +developers, or are binaries built from source by the admin of f-droid.org +using the tools on https://gitlab.com/u/fdroid. """ # As above, but for the archive repo. @@ -46,11 +46,11 @@ https://gitorious.org/f-droid. # repository, and no need to define the other archive_ values. archive_older = 3 archive_url = "https://f-droid.org/archive" -archive_name = "F-Droid Archive" +archive_name = "My First FDroid Archive Demo" archive_icon = "fdroid-icon.png" archive_description = """ -The archive repository of the F-Droid client. This contains older versions -of applications from the main repository. +The repository of older versions of applications from the main demo +repository. """ From 02721242486e7f455532e88b0057b0470bb9b498 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 4 Jun 2014 18:45:06 -0400 Subject: [PATCH 15/15] tests: fix --android-home test to actually work In this case, ANDROID_HOME is set to a fake, non-working version that will be detected by fdroid as an Android SDK install. It should use the path set by --android-home over the one in ANDROID_HOME, therefore if it uses the one in ANDROID_HOME, it won't work because it is a fake one. Only --android-home provides a working one. --- tests/run-tests | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/run-tests b/tests/run-tests index 49aadeb5..baa239ec 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -138,6 +138,11 @@ set -e #------------------------------------------------------------------------------# +# In this case, ANDROID_HOME is set to a fake, non-working version that will +# be detected by fdroid as an Android SDK install. It should use the path set +# by --android-home over the one in ANDROID_HOME, therefore if it uses the one +# in ANDROID_HOME, it won't work because it is a fake one. Only +# --android-home provides a working one. echo_header "setup a new repo from scratch with keystore and android-home set on cmd line" REPOROOT=`create_test_dir` @@ -148,7 +153,7 @@ STORED_ANDROID_HOME=$ANDROID_HOME unset ANDROID_HOME echo "ANDROID_HOME: $ANDROID_HOME" cd $REPOROOT -$fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME --no-prompt +$fdroid init --keystore $KEYSTORE --android-home $STORED_ANDROID_HOME --no-prompt test -e $KEYSTORE copy_apks_into_repo $REPOROOT $fdroid update --create-metadata