Merge branch 'build_tools_log' into 'master'

post correct build tool versions to wiki

Closes #267

See merge request !235

fdroidserver/build.py

@@ -243,8 +243,16 @@ def release_vm():
 
 
 # Note that 'force' here also implies test mode.
-def build_server(app, build, vcs, build_dir, output_dir, force):
+def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
-    """Do a build on the build server."""
+    """Do a build on the builder vm.
+
+    :param app: app metadata dict
+    :param build:
+    :param vcs: version control system controller object
+    :param build_dir: local source-code checkout of app
+    :param output_dir: target folder for the build result
+    :param force:
+    """
 
     global buildserverid
 
@@ -399,7 +407,6 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
         chan.exec_command('bash --login -c "' + cmdline + '"')
 
         output = bytes()
-        output += b'== Installed Android Tools ==\n\n'
         output += get_android_tools_version_log(build.ndk_path()).encode()
         while not chan.exit_status_ready():
             while chan.recv_ready():
@@ -417,6 +424,15 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
                 "Build.py failed on server for {0}:{1}".format(
                     app.id, build.versionName), str(output, 'utf-8'))
 
+        # Retreive logs...
+        toolsversion_log = common.get_toolsversion_logname(app, build)
+        try:
+            ftp.chdir(os.path.join(homedir, log_dir))
+            ftp.get(toolsversion_log, os.path.join(log_dir, toolsversion_log))
+            logging.debug('retrieved %s', toolsversion_log)
+        except Exception as e:
+            logging.warn('could not get %s from builder vm: %s' % (toolsversion_log, e))
+
         # Retrieve the built files...
         logging.info("Retrieving build output...")
         if force:
@@ -429,7 +445,7 @@ def build_server(app, build, vcs, build_dir, output_dir, force):
             ftp.get(apkfile, os.path.join(output_dir, apkfile))
             if not options.notarball:
                 ftp.get(tarball, os.path.join(output_dir, tarball))
-        except:
+        except Exception:
             raise BuildException(
                 "Build failed for %s:%s - missing output files".format(
                     app.id, build.versionName), output)
@@ -513,7 +529,7 @@ def get_metadata_from_apk(app, build, apkfile):
     return vercode, version
 
 
-def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
+def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
     """Do a build locally."""
 
     ndk_path = build.ndk_path()
@@ -532,6 +548,13 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
 
     common.set_FDroidPopen_env(build)
 
+    # create ..._toolsversion.log when running in builder vm
+    if onserver:
+        log_path = os.path.join(log_dir,
+                                common.get_toolsversion_logname(app, build))
+        with open(log_path, 'w') as f:
+            f.write(get_android_tools_version_log(build.ndk_path()))
+
     # Prepare the source code...
     root_dir, srclibpaths = common.prepare_source(vcs, app, build,
                                                   build_dir, srclib_dir,
@@ -565,7 +588,7 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
         if flavours == ['yes']:
             flavours = []
 
-        flavours_cmd = ''.join([capitalize_intact(f) for f in flavours])
+        flavours_cmd = ''.join([capitalize_intact(flav) for flav in flavours])
 
         gradletasks += ['assemble' + flavours_cmd + 'Release']
 
@@ -914,8 +937,9 @@ def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir,
                     os.path.join(output_dir, tarname))
 
 
-def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
+def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
-             tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh):
+             srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, test,
+             server, force, onserver, refresh):
     """
     Build a particular version of an application, if it needs building.
 
@@ -958,9 +982,9 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl
         # grabbing the source now.
         vcs.gotorevision(build.commit)
 
-        build_server(app, build, vcs, build_dir, output_dir, force)
+        build_server(app, build, vcs, build_dir, output_dir, log_dir, force)
     else:
-        build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
+        build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
     return True
 
 
@@ -992,7 +1016,7 @@ def get_android_tools_versions(ndk_path=None):
 
 def get_android_tools_version_log(ndk_path):
     '''get a list of the versions of all installed Android SDK/NDK components'''
-    log = ''
+    log = '== Installed Android Tools ==\n\n'
     components = get_android_tools_versions(ndk_path)
     for name, version in sorted(components):
         log += '* ' + name + ' (' + version + ')\n'
@@ -1151,8 +1175,9 @@ def main():
 
         for build in app.builds:
             wikilog = None
-            tools_version_log = '== Installed Android Tools ==\n\n'
+            tools_version_log = ''
-            tools_version_log += get_android_tools_version_log(build.ndk_path())
+            if not options.onserver:
+                tools_version_log = get_android_tools_version_log(build.ndk_path())
             try:
 
                 # For the first build of a particular app, we need to set up
@@ -1163,11 +1188,17 @@ def main():
                     first = False
 
                 logging.debug("Checking " + build.versionName)
-                if trybuild(app, build, build_dir, output_dir,
+                if trybuild(app, build, build_dir, output_dir, log_dir,
                             also_check_dir, srclib_dir, extlib_dir,
                             tmp_dir, repo_dir, vcs, options.test,
                             options.server, options.force,
                             options.onserver, options.refresh):
+                    toolslog = os.path.join(log_dir,
+                                            common.get_toolsversion_logname(app, build))
+                    if not options.onserver and os.path.exists(toolslog):
+                        with open(toolslog, 'r') as f:
+                            tools_version_log = ''.join(f.readlines())
+                        os.remove(toolslog)
 
                     if app.Binaries is not None:
                         # This is an app where we build from source, and
@@ -1234,8 +1265,8 @@ def main():
                     # Redirect from /lastbuild to the most recent build log
                     newpage = site.Pages[appid + '/lastbuild']
                     newpage.save('#REDIRECT [[' + lastbuildpage + ']]', summary='Update redirect')
-                except:
+                except Exception as e:
-                    logging.error("Error while attempting to publish build log")
+                    logging.error("Error while attempting to publish build log: %s" % e)
 
     for app in build_succeeded:
         logging.info("success: %s" % (app.id))

fdroidserver/common.py

@@ -552,6 +552,10 @@ def get_release_filename(app, build):
         return "%s_%s.apk" % (app.id, build.versionCode)
 
 
+def get_toolsversion_logname(app, build):
+    return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
+
+
 def getsrcname(app, build):
     return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
 

fdroidserver/dscanner.py

@@ -302,7 +302,7 @@ class DockerDriver(object):
                 msg = msg.format(c, t, int(c / t * 100), j)
                 sys.stdout.write(msg)
                 sys.stdout.flush()
-            except:
+            except Exception:
                 pass
         print("\nDONE!\n")
 

fdroidserver/update.py

@@ -300,8 +300,8 @@ def update_wiki(apps, sortedids, apks):
                 try:
                     newpage = site.Pages[pagename]
                     newpage.save(text, summary='Auto-created')
-                except:
+                except Exception as e:
-                    logging.error("...FAILED to create page '{0}'".format(pagename))
+                    logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 
     # Purge server cache to ensure counts are up to date
     site.pages['Repository Maintenance'].purge()
@@ -925,8 +925,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
                     f.write(get_icon_bytes(apkzip, iconsrc))
                 apk['icons'][density] = iconfilename
 
-            except:
+            except Exception as e:
-                logging.warn("Error retrieving icon file")
+                logging.warn("Error retrieving icon file: %s" % (e))
                 del apk['icons'][density]
                 del apk['icons_src'][density]
                 empty_densities.append(density)
@@ -982,8 +982,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
                 im.thumbnail((size, size), Image.ANTIALIAS)
                 im.save(iconpath, "PNG")
                 empty_densities.remove(density)
-            except:
+            except Exception as e:
-                logging.warning("Invalid image file at %s" % last_iconpath)
+                logging.warning("Invalid image file at %s: %s" % (last_iconpath, e))
             finally:
                 if fp:
                     fp.close()