diff --git a/fdroidserver/build.py b/fdroidserver/build.py index c889e972..4029b08f 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -243,8 +243,16 @@ def release_vm(): # Note that 'force' here also implies test mode. -def build_server(app, build, vcs, build_dir, output_dir, force): - """Do a build on the build server.""" +def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): + """Do a build on the builder vm. + + :param app: app metadata dict + :param build: + :param vcs: version control system controller object + :param build_dir: local source-code checkout of app + :param output_dir: target folder for the build result + :param force: + """ global buildserverid @@ -399,7 +407,6 @@ def build_server(app, build, vcs, build_dir, output_dir, force): chan.exec_command('bash --login -c "' + cmdline + '"') output = bytes() - output += b'== Installed Android Tools ==\n\n' output += get_android_tools_version_log(build.ndk_path()).encode() while not chan.exit_status_ready(): while chan.recv_ready(): @@ -417,6 +424,15 @@ def build_server(app, build, vcs, build_dir, output_dir, force): "Build.py failed on server for {0}:{1}".format( app.id, build.versionName), str(output, 'utf-8')) + # Retreive logs... + toolsversion_log = common.get_toolsversion_logname(app, build) + try: + ftp.chdir(os.path.join(homedir, log_dir)) + ftp.get(toolsversion_log, os.path.join(log_dir, toolsversion_log)) + logging.debug('retrieved %s', toolsversion_log) + except Exception as e: + logging.warn('could not get %s from builder vm: %s' % (toolsversion_log, e)) + # Retrieve the built files... logging.info("Retrieving build output...") if force: @@ -429,7 +445,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force): ftp.get(apkfile, os.path.join(output_dir, apkfile)) if not options.notarball: ftp.get(tarball, os.path.join(output_dir, tarball)) - except: + except Exception: raise BuildException( "Build failed for %s:%s - missing output files".format( app.id, build.versionName), output) @@ -513,7 +529,7 @@ def get_metadata_from_apk(app, build, apkfile): return vercode, version -def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): +def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): """Do a build locally.""" ndk_path = build.ndk_path() @@ -532,6 +548,13 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, common.set_FDroidPopen_env(build) + # create ..._toolsversion.log when running in builder vm + if onserver: + log_path = os.path.join(log_dir, + common.get_toolsversion_logname(app, build)) + with open(log_path, 'w') as f: + f.write(get_android_tools_version_log(build.ndk_path())) + # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, build, build_dir, srclib_dir, @@ -565,7 +588,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, if flavours == ['yes']: flavours = [] - flavours_cmd = ''.join([capitalize_intact(f) for f in flavours]) + flavours_cmd = ''.join([capitalize_intact(flav) for flav in flavours]) gradletasks += ['assemble' + flavours_cmd + 'Release'] @@ -914,8 +937,9 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, os.path.join(output_dir, tarname)) -def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, - tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh): +def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, + srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, test, + server, force, onserver, refresh): """ Build a particular version of an application, if it needs building. @@ -958,9 +982,9 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl # grabbing the source now. vcs.gotorevision(build.commit) - build_server(app, build, vcs, build_dir, output_dir, force) + build_server(app, build, vcs, build_dir, output_dir, log_dir, force) else: - build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh) + build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh) return True @@ -992,7 +1016,7 @@ def get_android_tools_versions(ndk_path=None): def get_android_tools_version_log(ndk_path): '''get a list of the versions of all installed Android SDK/NDK components''' - log = '' + log = '== Installed Android Tools ==\n\n' components = get_android_tools_versions(ndk_path) for name, version in sorted(components): log += '* ' + name + ' (' + version + ')\n' @@ -1151,8 +1175,9 @@ def main(): for build in app.builds: wikilog = None - tools_version_log = '== Installed Android Tools ==\n\n' - tools_version_log += get_android_tools_version_log(build.ndk_path()) + tools_version_log = '' + if not options.onserver: + tools_version_log = get_android_tools_version_log(build.ndk_path()) try: # For the first build of a particular app, we need to set up @@ -1163,11 +1188,17 @@ def main(): first = False logging.debug("Checking " + build.versionName) - if trybuild(app, build, build_dir, output_dir, + if trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver, options.refresh): + toolslog = os.path.join(log_dir, + common.get_toolsversion_logname(app, build)) + if not options.onserver and os.path.exists(toolslog): + with open(toolslog, 'r') as f: + tools_version_log = ''.join(f.readlines()) + os.remove(toolslog) if app.Binaries is not None: # This is an app where we build from source, and @@ -1234,8 +1265,8 @@ def main(): # Redirect from /lastbuild to the most recent build log newpage = site.Pages[appid + '/lastbuild'] newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect') - except: - logging.error("Error while attempting to publish build log") + except Exception as e: + logging.error("Error while attempting to publish build log: %s" % e) for app in build_succeeded: logging.info("success: %s" % (app.id)) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index cae27803..bb0d7b81 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -552,6 +552,10 @@ def get_release_filename(app, build): return "%s_%s.apk" % (app.id, build.versionCode) +def get_toolsversion_logname(app, build): + return "%s_%s_toolsversion.log" % (app.id, build.versionCode) + + def getsrcname(app, build): return "%s_%s_src.tar.gz" % (app.id, build.versionCode) diff --git a/fdroidserver/dscanner.py b/fdroidserver/dscanner.py index 8e224bdb..10e98b31 100644 --- a/fdroidserver/dscanner.py +++ b/fdroidserver/dscanner.py @@ -302,7 +302,7 @@ class DockerDriver(object): msg = msg.format(c, t, int(c / t * 100), j) sys.stdout.write(msg) sys.stdout.flush() - except: + except Exception: pass print("\nDONE!\n") diff --git a/fdroidserver/update.py b/fdroidserver/update.py index eb92e91b..7ad41b9a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -300,8 +300,8 @@ def update_wiki(apps, sortedids, apks): try: newpage = site.Pages[pagename] newpage.save(text, summary='Auto-created') - except: - logging.error("...FAILED to create page '{0}'".format(pagename)) + except Exception as e: + logging.error("...FAILED to create page '{0}': {1}".format(pagename, e)) # Purge server cache to ensure counts are up to date site.pages['Repository Maintenance'].purge() @@ -925,8 +925,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): f.write(get_icon_bytes(apkzip, iconsrc)) apk['icons'][density] = iconfilename - except: - logging.warn("Error retrieving icon file") + except Exception as e: + logging.warn("Error retrieving icon file: %s" % (e)) del apk['icons'][density] del apk['icons_src'][density] empty_densities.append(density) @@ -982,8 +982,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): im.thumbnail((size, size), Image.ANTIALIAS) im.save(iconpath, "PNG") empty_densities.remove(density) - except: - logging.warning("Invalid image file at %s" % last_iconpath) + except Exception as e: + logging.warning("Invalid image file at %s: %s" % (last_iconpath, e)) finally: if fp: fp.close()