diff --git a/fdroid b/fdroid index 74339df0..a66df6b1 100755 --- a/fdroid +++ b/fdroid @@ -18,6 +18,7 @@ # along with this program. If not, see . import sys +import os import logging import fdroidserver.common @@ -73,7 +74,6 @@ def main(): print_help() sys.exit(0) elif command == '--version': - import os.path output = _('no version info found!') cmddir = os.path.realpath(os.path.dirname(__file__)) moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..') @@ -143,7 +143,9 @@ def main(): sys.exit(1) except KeyboardInterrupt: print('') - sys.exit(1) + sys.stdout.flush() + sys.stderr.flush() + os._exit(1) # These should only be unexpected crashes due to bugs in the code # str(e) often doesn't contain a reason, so just show the backtrace except Exception as e: diff --git a/fdroidserver/build.py b/fdroidserver/build.py index a431dd52..05700cf7 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -25,6 +25,7 @@ import re import resource import sys import tarfile +import threading import traceback import time import requests @@ -239,9 +240,12 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): logging.info("...getting exit status") returncode = chan.recv_exit_status() if returncode != 0: - raise BuildException( - "Build.py failed on server for {0}:{1}".format( - app.id, build.versionName), None if options.verbose else str(output, 'utf-8')) + if timeout_event.is_set(): + message = "Timeout exceeded! Build VM force-stopped for {0}:{1}" + else: + message = "Build.py failed on server for {0}:{1}" + raise BuildException(message.format(app.id, build.versionName), + None if options.verbose else str(output, 'utf-8')) # Retreive logs... toolsversion_log = common.get_toolsversion_logname(app, build) @@ -978,6 +982,14 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, return True +def force_halt_build(): + """Halt the currently running Vagrant VM, to be called from a Timer""" + logging.error(_('Force halting build after timeout!')) + timeout_event.set() + vm = vmtools.get_build_vm('builder') + vm.halt() + + def parse_commandline(): """Parse the command line. Returns options, parser.""" @@ -1029,6 +1041,7 @@ config = None buildserverid = None fdroidserverid = None start_timestamp = time.gmtime() +timeout_event = threading.Event() def main(): @@ -1138,15 +1151,23 @@ def main(): # Build applications... failed_apps = {} build_succeeded = [] - max_apps_per_run = 50 + # Only build for 12 hours, then stop gracefully + endtime = time.time() + 12 * 60 * 60 + max_build_time_reached = False for appid, app in apps.items(): - max_apps_per_run -= 1 - if max_apps_per_run < 1: - break first = True for build in app.builds: + if time.time() > endtime: + max_build_time_reached = True + break + if options.server: # enable watchdog timer + timer = threading.Timer(7200, force_halt_build) + timer.start() + else: + timer = None + wikilog = None build_starttime = common.get_wiki_timestamp() tools_version_log = '' @@ -1287,6 +1308,13 @@ def main(): except Exception as e: logging.error("Error while attempting to publish build log: %s" % e) + if timer: + timer.cancel() # kill the watchdog timer + + if max_build_time_reached: + logging.info("Stopping after global build timeout...") + break + for app in build_succeeded: logging.info("success: %s" % (app.id)) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 6671a3eb..33544ac5 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -29,6 +29,9 @@ from .common import FDroidException from logging import getLogger from fdroidserver import _ +import threading + +lock = threading.Lock() logger = getLogger('fdroidserver-vmtools') @@ -175,7 +178,6 @@ class FDroidBuildVm(): This is intended to be a hypervisor independant, fault tolerant wrapper around the vagrant functions we use. """ - def __init__(self, srvdir): """Create new server class. """ @@ -191,21 +193,27 @@ class FDroidBuildVm(): self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm) def up(self, provision=True): - try: - self.vgrnt.up(provision=provision) - self.srvuuid = self._vagrant_fetch_uuid() - except subprocess.CalledProcessError as e: - raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e + global lock + with lock: + try: + self.vgrnt.up(provision=provision) + self.srvuuid = self._vagrant_fetch_uuid() + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e def suspend(self): - logger.info('suspending buildserver') - try: - self.vgrnt.suspend() - except subprocess.CalledProcessError as e: - raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e + global lock + with lock: + logger.info('suspending buildserver') + try: + self.vgrnt.suspend() + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e def halt(self): - self.vgrnt.halt(force=True) + global lock + with lock: + self.vgrnt.halt(force=True) def destroy(self): """Remove every trace of this VM from the system.