From a14f82c49e748aa2efaf748eb076dbf107ab83db Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Jan 2015 18:54:01 +0100 Subject: [PATCH 1/7] prevent `fdroid publish` from repeatedly downloading the developer binary wget's --continue makes wget not overwrite the existing file, or create a new version with a .1 extension. Instead it tries to finish an incomplete download, or does nothing if the download is complete and matching. --- fdroidserver/publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 48d0503e..fc54ee4f 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -144,7 +144,7 @@ def main(): # Grab the binary from where the developer publishes it... logging.info("...retrieving " + url) srcapk = os.path.join(tmp_dir, url.split('/')[-1]) - p = FDroidPopen(['wget', '-nv', url], cwd=tmp_dir) + p = FDroidPopen(['wget', '-nv', '--continue', url], cwd=tmp_dir) if p.returncode != 0 or not os.path.exists(srcapk): logging.error("...failed to retrieve " + url + " - publish skipped") From b53746dc7ba3711ada12296a92ba13edae286293 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Jan 2015 19:55:26 +0100 Subject: [PATCH 2/7] make resulting dirs of compare_apks() have clearer dir names This makes it a lot easier to remember which APK is which when trying to make sense of the differences. --- fdroidserver/common.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 3a95445a..a7b380fb 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1879,24 +1879,24 @@ def compare_apks(apk1, apk2, tmp_dir): trying to do the comparison. """ - thisdir = os.path.join(tmp_dir, 'this_apk') - thatdir = os.path.join(tmp_dir, 'that_apk') - for d in [thisdir, thatdir]: + badchars = re.compile('''[/ :;'"]''') + apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk + apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk + for d in [apk1dir, apk2dir]: if os.path.exists(d): shutil.rmtree(d) os.mkdir(d) if subprocess.call(['jar', 'xf', os.path.abspath(apk1)], - cwd=thisdir) != 0: + cwd=apk1dir) != 0: return("Failed to unpack " + apk1) if subprocess.call(['jar', 'xf', os.path.abspath(apk2)], - cwd=thatdir) != 0: + cwd=apk2dir) != 0: return("Failed to unpack " + apk2) - p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir, - output=False) + p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False) lines = p.output.splitlines() if len(lines) != 1 or 'META-INF' not in lines[0]: return("Unexpected diff output - " + p.output) From 74a785a888b497ff10edd519d7d92def439331c3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Jan 2015 19:56:55 +0100 Subject: [PATCH 3/7] uncompress using `jar xf` in a folder with the same name This makes it easier to compare the whole APK folders when there are multiple methods of uncompressing the APK. --- fdroidserver/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a7b380fb..c8ed3756 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1886,14 +1886,15 @@ def compare_apks(apk1, apk2, tmp_dir): if os.path.exists(d): shutil.rmtree(d) os.mkdir(d) + os.mkdir(os.path.join(d, 'jar-xf')) if subprocess.call(['jar', 'xf', os.path.abspath(apk1)], - cwd=apk1dir) != 0: + cwd=os.path.join(apk1dir, 'jar-xf')) != 0: return("Failed to unpack " + apk1) if subprocess.call(['jar', 'xf', os.path.abspath(apk2)], - cwd=apk2dir) != 0: + cwd=os.path.join(apk2dir, 'jar-xf')) != 0: return("Failed to unpack " + apk2) p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False) From 6bc6ba05b2d089f27655056491d687789e894a02 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Jan 2015 19:57:15 +0100 Subject: [PATCH 4/7] fix typo in error message --- fdroidserver/publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index fc54ee4f..b5ea2a75 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -153,7 +153,7 @@ def main(): # Compare our unsigned one with the downloaded one... compare_result = common.compare_apks(srcapk, apkfile, tmp_dir) if compare_result: - logging.error("...verfication failed - publish skipped : " + logging.error("...verification failed - publish skipped : " + compare_result) continue From fc4679cebba468a7491301959cfd71504ac20aea Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Jan 2015 20:08:15 +0100 Subject: [PATCH 5/7] if `apktool` is available, use it to decompress APKs when verifying apktool decompiles the binary XML to regular XML, so it is much easier to read for differences. --- fdroidserver/common.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index c8ed3756..bfeb544d 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1897,6 +1897,19 @@ def compare_apks(apk1, apk2, tmp_dir): cwd=os.path.join(apk2dir, 'jar-xf')) != 0: return("Failed to unpack " + apk2) + # try to find apktool in the path, if it hasn't been manually configed + if not 'apktool' in config: + tmp = find_command('apktool') + if not tmp is None: + config['apktool'] = tmp + if 'apktool' in config: + if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'], + cwd=apk1dir) != 0: + return("Failed to unpack " + apk1) + if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'], + cwd=apk2dir) != 0: + return("Failed to unpack " + apk2) + p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False) lines = p.output.splitlines() if len(lines) != 1 or 'META-INF' not in lines[0]: @@ -1904,3 +1917,23 @@ def compare_apks(apk1, apk2, tmp_dir): # If we get here, it seems like they're the same! return None + + +def find_command(command): + '''find the full path of a command, or None if it can't be found in the PATH''' + + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(command) + if fpath: + if is_exe(command): + return command + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, command) + if is_exe(exe_file): + return exe_file + + return None From 2458577209b0cafbf0f12153abc14942e5953332 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Jan 2015 20:09:03 +0100 Subject: [PATCH 6/7] if `meld` is available, use it to show differences on failed APK compare meld is much easier to browse when comparing directory trees. --- fdroidserver/common.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index bfeb544d..2ea43885 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1913,6 +1913,9 @@ def compare_apks(apk1, apk2, tmp_dir): p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False) lines = p.output.splitlines() if len(lines) != 1 or 'META-INF' not in lines[0]: + meld = find_command('meld') + if not meld is None: + p = FDroidPopen(['meld', apk1dir, apk2dir], output=False) return("Unexpected diff output - " + p.output) # If we get here, it seems like they're the same! From 4e6865543794c252623fcc343e9171b2cef0b4bf Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 12 Jan 2015 10:51:54 +0100 Subject: [PATCH 7/7] if a build did verify against Binaries:, delete the comparison dirs If the comparison between the provided APK and the built APK is verified, then there is no need to keep the old comparison files around. On big build servers like f-droid.org, there will be thousands of leftover dirs from all the builds, so this is necessary. --- fdroidserver/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 2ea43885..b7986d20 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1918,6 +1918,10 @@ def compare_apks(apk1, apk2, tmp_dir): p = FDroidPopen(['meld', apk1dir, apk2dir], output=False) return("Unexpected diff output - " + p.output) + # since everything verifies, delete the comparison to keep cruft down + shutil.rmtree(apk1dir) + shutil.rmtree(apk2dir) + # If we get here, it seems like they're the same! return None