From bb49123118dee0760e8c067f150fda1a407d719d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 21 Mar 2017 23:51:15 +0100 Subject: [PATCH 1/5] post correct build tool versions to wiki --- fdroidserver/build.py | 54 ++++++++++++++++++++++++++++++++++-------- fdroidserver/common.py | 4 ++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index c889e972..eb4aab94 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -243,8 +243,16 @@ def release_vm(): # Note that 'force' here also implies test mode. -def build_server(app, build, vcs, build_dir, output_dir, force): - """Do a build on the build server.""" +def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): + """Do a build on the builder vm. + + :param app: app metadata dict + :param build: + :param vcs: version control system controller object + :param build_dir: local source-code checkout of app + :param output_dir: target folder for the build result + :param force: + """ global buildserverid @@ -417,6 +425,15 @@ def build_server(app, build, vcs, build_dir, output_dir, force): "Build.py failed on server for {0}:{1}".format( app.id, build.versionName), str(output, 'utf-8')) + # Retreive logs... + toolsversion_log = common.get_toolsversion_logname(app, build) + try: + ftp.chdir(os.path.join(homedir, log_dir)) + ftp.get(toolsversion_log, os.path.join(log_dir, toolsversion_log)) + logging.debug('retrieved %s', toolsversion_log) + except: + logging.warn('could not get %s from builder vm.' % (toolsversion_log)) + # Retrieve the built files... logging.info("Retrieving build output...") if force: @@ -513,7 +530,7 @@ def get_metadata_from_apk(app, build, apkfile): return vercode, version -def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): +def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): """Do a build locally.""" ndk_path = build.ndk_path() @@ -532,6 +549,14 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, common.set_FDroidPopen_env(build) + # create ..._toolsversion.log when running in builder vm + if onserver: + log_path = os.path.join(log_dir, + common.get_toolsversion_logname(app, build)) + with open(log_path, 'w') as f: + f.write('== Installed Android Tools ==\n\n') + f.write(get_android_tools_version_log(build.ndk_path())) + # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, build, build_dir, srclib_dir, @@ -914,8 +939,9 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, os.path.join(output_dir, tarname)) -def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, - tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh): +def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, + srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, test, + server, force, onserver, refresh): """ Build a particular version of an application, if it needs building. @@ -958,9 +984,9 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl # grabbing the source now. vcs.gotorevision(build.commit) - build_server(app, build, vcs, build_dir, output_dir, force) + build_server(app, build, vcs, build_dir, output_dir, log_dir, force) else: - build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh) + build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh) return True @@ -1151,8 +1177,10 @@ def main(): for build in app.builds: wikilog = None - tools_version_log = '== Installed Android Tools ==\n\n' - tools_version_log += get_android_tools_version_log(build.ndk_path()) + tools_version_log = '' + if not options.onserver: + tools_version_log = '== Installed Android Tools ==\n\n' + tools_version_log += get_android_tools_version_log(build.ndk_path()) try: # For the first build of a particular app, we need to set up @@ -1163,11 +1191,17 @@ def main(): first = False logging.debug("Checking " + build.versionName) - if trybuild(app, build, build_dir, output_dir, + if trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver, options.refresh): + toolslog = os.path.join(log_dir, + common.get_toolsversion_logname(app, build)) + if not options.onserver and os.path.exists(toolslog): + with open(toolslog, 'r') as f: + tools_version_log = ''.join(f.readlines()) + os.remove(toolslog) if app.Binaries is not None: # This is an app where we build from source, and diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 31c97380..d08a432c 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -552,6 +552,10 @@ def get_release_filename(app, build): return "%s_%s.apk" % (app.id, build.versionCode) +def get_toolsversion_logname(app, build): + return "%s_%s_toolsversion.log" % (app.id, build.versionCode) + + def getsrcname(app, build): return "%s_%s_src.tar.gz" % (app.id, build.versionCode) From 83fe2bda4d140821ad0c7e48f56d28aafb6c7971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 22 Mar 2017 14:01:32 +0100 Subject: [PATCH 2/5] fix exception handling in build.py --- fdroidserver/build.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index eb4aab94..901b14ad 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -431,8 +431,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.chdir(os.path.join(homedir, log_dir)) ftp.get(toolsversion_log, os.path.join(log_dir, toolsversion_log)) logging.debug('retrieved %s', toolsversion_log) - except: - logging.warn('could not get %s from builder vm.' % (toolsversion_log)) + except Exception as e: + logging.warn('could not get %s from builder vm: %s' % (toolsversion_log, e)) # Retrieve the built files... logging.info("Retrieving build output...") @@ -446,7 +446,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.get(apkfile, os.path.join(output_dir, apkfile)) if not options.notarball: ftp.get(tarball, os.path.join(output_dir, tarball)) - except: + except Exception: raise BuildException( "Build failed for %s:%s - missing output files".format( app.id, build.versionName), output) @@ -1268,8 +1268,8 @@ def main(): # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') - except: - logging.error("Error while attempting to publish build log") + except Exception as e: + logging.error("Error while attempting to publish build log: %s" % e) for app in build_succeeded: logging.info("success: %s" % (app.id)) From c15f02ed680432bb055db47f50e799707bbf2e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 22 Mar 2017 14:07:54 +0100 Subject: [PATCH 3/5] fix exception handling in dscanner and update.py --- fdroidserver/dscanner.py | 2 +- fdroidserver/update.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fdroidserver/dscanner.py b/fdroidserver/dscanner.py index 8e224bdb..10e98b31 100644 --- a/fdroidserver/dscanner.py +++ b/fdroidserver/dscanner.py @@ -302,7 +302,7 @@ class DockerDriver(object): msg = msg.format(c, t, int(c / t * 100), j) sys.stdout.write(msg) sys.stdout.flush() - except: + except Exception: pass print("\nDONE!\n") diff --git a/fdroidserver/update.py b/fdroidserver/update.py index eb92e91b..7ad41b9a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -300,8 +300,8 @@ def update_wiki(apps, sortedids, apks): try: newpage = site.Pages[pagename] newpage.save(text, summary='Auto-created') - except: - logging.error("...FAILED to create page '{0}'".format(pagename)) + except Exception as e: + logging.error("...FAILED to create page '{0}': {1}".format(pagename, e)) # Purge server cache to ensure counts are up to date site.pages['Repository Maintenance'].purge() @@ -925,8 +925,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): f.write(get_icon_bytes(apkzip, iconsrc)) apk['icons'][density] = iconfilename - except: - logging.warn("Error retrieving icon file") + except Exception as e: + logging.warn("Error retrieving icon file: %s" % (e)) del apk['icons'][density] del apk['icons_src'][density] empty_densities.append(density) @@ -982,8 +982,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): im.thumbnail((size, size), Image.ANTIALIAS) im.save(iconpath, "PNG") empty_densities.remove(density) - except: - logging.warning("Invalid image file at %s" % last_iconpath) + except Exception as e: + logging.warning("Invalid image file at %s: %s" % (last_iconpath, e)) finally: if fp: fp.close() From 0fda23ef8068c022fdb3c5d5b0e02c7e367c5c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 22 Mar 2017 14:20:54 +0100 Subject: [PATCH 4/5] fix local var name overloading when iterating build flavours --- fdroidserver/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 901b14ad..ffd24bc9 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -590,7 +590,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext if flavours == ['yes']: flavours = [] - flavours_cmd = ''.join([capitalize_intact(f) for f in flavours]) + flavours_cmd = ''.join([capitalize_intact(flav) for flav in flavours]) gradletasks += ['assemble' + flavours_cmd + 'Release'] From 843595eb339fd9e37f2cefb36255de2098441241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 22 Mar 2017 14:56:26 +0100 Subject: [PATCH 5/5] moved version android tools version log header to corresponding function --- fdroidserver/build.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index ffd24bc9..4029b08f 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -407,7 +407,6 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): chan.exec_command('bash --login -c "' + cmdline + '"') output = bytes() - output += b'== Installed Android Tools ==\n\n' output += get_android_tools_version_log(build.ndk_path()).encode() while not chan.exit_status_ready(): while chan.recv_ready(): @@ -554,7 +553,6 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext log_path = os.path.join(log_dir, common.get_toolsversion_logname(app, build)) with open(log_path, 'w') as f: - f.write('== Installed Android Tools ==\n\n') f.write(get_android_tools_version_log(build.ndk_path())) # Prepare the source code... @@ -1018,7 +1016,7 @@ def get_android_tools_versions(ndk_path=None): def get_android_tools_version_log(ndk_path): '''get a list of the versions of all installed Android SDK/NDK components''' - log = '' + log = '== Installed Android Tools ==\n\n' components = get_android_tools_versions(ndk_path) for name, version in sorted(components): log += '* ' + name + ' (' + version + ')\n' @@ -1179,8 +1177,7 @@ def main(): wikilog = None tools_version_log = '' if not options.onserver: - tools_version_log = '== Installed Android Tools ==\n\n' - tools_version_log += get_android_tools_version_log(build.ndk_path()) + tools_version_log = get_android_tools_version_log(build.ndk_path()) try: # For the first build of a particular app, we need to set up