From 874565fb730ad6d36059c47a0ab408c1ead61b46 Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Thu, 16 Jan 2014 10:17:22 +0000 Subject: [PATCH] Don't split stdout/stderr For the most part, doing this just makes build output incomprehensible, with errors appearing out of context. --- fdroidserver/build.py | 23 ++++++--------- fdroidserver/common.py | 62 ++++++++++++++--------------------------- fdroidserver/init.py | 2 +- fdroidserver/install.py | 2 +- 4 files changed, 31 insertions(+), 58 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 7204d57d..b657da48 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -326,6 +326,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force): # Execute the build script... print "Starting build..." chan = sshs.get_transport().open_session() + chan.get_pty() cmdline = 'python build.py --on-server' if force: cmdline += ' --force --test' @@ -334,12 +335,9 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force): cmdline += " %s:%s" % (app['id'], thisbuild['vercode']) chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"') output = '' - error = '' while not chan.exit_status_ready(): while chan.recv_ready(): output += chan.recv(1024) - while chan.recv_stderr_ready(): - error += chan.recv_stderr(1024) time.sleep(0.1) print "...getting exit status" returncode = chan.recv_exit_status() @@ -348,13 +346,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force): if len(get) == 0: break output += get - while True: - get = chan.recv_stderr(1024) - if len(get) == 0: - break - error += get if returncode != 0: - raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output, error) + raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output) # Retrieve the built files... print "Retrieving build output..." @@ -368,7 +361,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force): ftp.get(apkfile, os.path.join(output_dir, apkfile)) ftp.get(tarball, os.path.join(output_dir, tarball)) except: - raise BuildException("Build failed for %s:%s - missing output files" % (app['id'], thisbuild['version']), output, error) + raise BuildException("Build failed for %s:%s - missing output files" % (app['id'], thisbuild['version']), output) ftp.close() finally: @@ -432,7 +425,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if p is not None and p.returncode != 0: raise BuildException("Error cleaning %s:%s" % - (app['id'], thisbuild['version']), p.stdout, p.stderr) + (app['id'], thisbuild['version']), p.stdout) # Scan before building... print "Scanning source for common problems..." @@ -468,7 +461,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if p.returncode != 0: raise BuildException("Error running build command for %s:%s" % - (app['id'], thisbuild['version']), p.stdout, p.stderr) + (app['id'], thisbuild['version']), p.stdout) # Build native stuff if required... if thisbuild.get('buildjni') not in (None, 'no'): @@ -495,7 +488,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d del manifest_text p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d)) if p.returncode != 0: - raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) + raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout) p = None # Build the release... @@ -635,7 +628,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d bindir = os.path.join(root_dir, 'bin') if p.returncode != 0: - raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) + raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout) print "Successfully built version " + thisbuild['version'] + ' of ' + app['id'] # Find the apk name in the output... @@ -965,7 +958,7 @@ def main(): if len(txt) > 8192: txt = txt[-8192:] txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt - newpage.save(wikilog, summary='Build log') + newpage.save(txt, summary='Build log') except: print "Error while attempting to publish build log" diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5dc84fc2..449105a0 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -749,31 +749,23 @@ def parse_androidmanifests(paths): return (max_version, max_vercode, max_package) class BuildException(Exception): - def __init__(self, value, stdout = None, stderr = None): + def __init__(self, value, detail = None): self.value = value - self.stdout = stdout - self.stderr = stderr + self.detail = detail def get_wikitext(self): ret = repr(self.value) + "\n" - if self.stdout: - ret += "=stdout=\n" + if self.detail: + ret += "=detail=\n" ret += "
\n"
-            ret += str(self.stdout)
-            ret += "
\n" - if self.stderr: - ret += "=stderr=\n" - ret += "
\n"
-            ret += str(self.stderr)
+            ret += str(self.detail)
             ret += "
\n" return ret def __str__(self): ret = repr(self.value) - if self.stdout: - ret += "\n==== stdout begin ====\n%s\n==== stdout end ====" % self.stdout.strip() - if self.stderr: - ret += "\n==== stderr begin ====\n%s\n==== stderr end ====" % self.stderr.strip() + if self.detail: + ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip() return ret class VCSException(Exception): @@ -855,7 +847,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, target=None, p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir) if p.returncode != 0: raise BuildException("Error running prepare command for srclib %s" - % name, p.stdout, p.stderr) + % name, p.stdout) if srclib["Update Project"] == "Yes": print "Updating srclib %s at path %s" % (name, libdir) @@ -867,10 +859,9 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, target=None, # Check to see whether an error was returned without a proper exit # code (this is the case for the 'no target set or target invalid' # error) - if p.returncode != 0 or (p.stderr != "" and - p.stderr.startswith("Error: ")): + if p.returncode != 0 or p.stdout.startswith("Error: "): raise BuildException("Failed to update srclib project {0}" - .format(name), p.stdout, p.stderr) + .format(name), p.stdout) remove_signing_keys(libdir) @@ -926,7 +917,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running init command for %s:%s" % - (app['id'], build['version']), p.stdout, p.stderr) + (app['id'], build['version']), p.stdout) # Generate (or update) the ant build file, build.xml... updatemode = build.get('update', 'auto') @@ -963,10 +954,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Check to see whether an error was returned without a proper exit # code (this is the case for the 'no target set or target invalid' # error) - if p.returncode != 0 or (p.stderr != "" and - p.stderr.startswith("Error: ")): + if p.returncode != 0 or p.stdout.startswith("Error: "): raise BuildException("Failed to update project at %s" % d, - p.stdout, p.stderr) + p.stdout) # Update the local.properties file... localprops = [ os.path.join(build_dir, 'local.properties') ] @@ -1178,7 +1168,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running prebuild command for %s:%s" % - (app['id'], build['version']), p.stdout, p.stderr) + (app['id'], build['version']), p.stdout) return (root_dir, srclibpaths) @@ -1413,9 +1403,11 @@ class PopenResult: def FDroidPopen(commands, cwd=None): """ - Runs a command the FDroid way and returns return code and output + Run a command and capture the output. - :param commands and cwd like in subprocess.Popen + :param commands: command and argument list like in subprocess.Popen + :param cwd: optionally specifies a working directory + :returns: A PopenResult. """ if options.verbose: @@ -1425,18 +1417,14 @@ def FDroidPopen(commands, cwd=None): result = PopenResult() p = subprocess.Popen(commands, cwd=cwd, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout_queue = Queue.Queue() stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue) stdout_reader.start() - stderr_queue = Queue.Queue() - stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue) - stderr_reader.start() - # Check the queues for output (until there is no more to get) - while not stdout_reader.eof() or not stderr_reader.eof(): - # Show what we received from standard output + # Check the queue for output (until there is no more to get) + while not stdout_reader.eof(): while not stdout_queue.empty(): line = stdout_queue.get() if options.verbose: @@ -1445,14 +1433,6 @@ def FDroidPopen(commands, cwd=None): sys.stdout.flush() result.stdout += line - # Show what we received from standard error - while not stderr_queue.empty(): - line = stderr_queue.get() - if options.verbose: - # Output directly to console - sys.stderr.write(line) - sys.stderr.flush() - result.stderr += line time.sleep(0.1) p.communicate() diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 26bc897d..dea189bd 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -65,7 +65,7 @@ def genkey(keystore, repo_keyalias, password, keydname): '-storepass', password, '-keypass', password, '-dname', keydname]) if p.returncode != 0: - raise BuildException("Failed to generate key", p.stdout, p.stderr) + raise BuildException("Failed to generate key", p.stdout) # now show the lovely key that was just generated p = subprocess.Popen(['keytool', '-list', '-v', '-keystore', keystore, '-alias', repo_keyalias], diff --git a/fdroidserver/install.py b/fdroidserver/install.py index 85d6e34a..66be0663 100644 --- a/fdroidserver/install.py +++ b/fdroidserver/install.py @@ -32,7 +32,7 @@ config = None def devices(): p = FDroidPopen(["adb", "devices"]) if p.returncode != 0: - raise Exception("An error occured when finding devices: %s" % p.stderr) + raise Exception("An error occured when finding devices: %s" % p.stdout) lines = p.stdout.splitlines() if lines[0].startswith('* daemon not running'): lines = lines[2:]