Merge branch 'build_timeout' into 'master'

Build timeout

See merge request fdroid/fdroidserver!437

fdroid

@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import sys
+import os
 import logging
 
 import fdroidserver.common
@@ -73,7 +74,6 @@ def main():
             print_help()
             sys.exit(0)
         elif command == '--version':
-            import os.path
             output = _('no version info found!')
             cmddir = os.path.realpath(os.path.dirname(__file__))
             moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..')
@@ -143,7 +143,9 @@ def main():
         sys.exit(1)
     except KeyboardInterrupt:
         print('')
-        sys.exit(1)
+        sys.stdout.flush()
+        sys.stderr.flush()
+        os._exit(1)
     # These should only be unexpected crashes due to bugs in the code
     # str(e) often doesn't contain a reason, so just show the backtrace
     except Exception as e:

fdroidserver/build.py

@@ -25,6 +25,7 @@ import re
 import resource
 import sys
 import tarfile
+import threading
 import traceback
 import time
 import requests
@@ -239,9 +240,12 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         logging.info("...getting exit status")
         returncode = chan.recv_exit_status()
         if returncode != 0:
-            raise BuildException(
+            if timeout_event.is_set():
-                "Build.py failed on server for {0}:{1}".format(
+                message = "Timeout exceeded! Build VM force-stopped for {0}:{1}"
-                    app.id, build.versionName), None if options.verbose else str(output, 'utf-8'))
+            else:
+                message = "Build.py failed on server for {0}:{1}"
+            raise BuildException(message.format(app.id, build.versionName),
+                                 None if options.verbose else str(output, 'utf-8'))
 
         # Retreive logs...
         toolsversion_log = common.get_toolsversion_logname(app, build)
@@ -978,6 +982,14 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
     return True
 
 
+def force_halt_build():
+    """Halt the currently running Vagrant VM, to be called from a Timer"""
+    logging.error(_('Force halting build after timeout!'))
+    timeout_event.set()
+    vm = vmtools.get_build_vm('builder')
+    vm.halt()
+
+
 def parse_commandline():
     """Parse the command line. Returns options, parser."""
 
@@ -1029,6 +1041,7 @@ config = None
 buildserverid = None
 fdroidserverid = None
 start_timestamp = time.gmtime()
+timeout_event = threading.Event()
 
 
 def main():
@@ -1138,15 +1151,23 @@ def main():
     # Build applications...
     failed_apps = {}
     build_succeeded = []
-    max_apps_per_run = 50
+    # Only build for 12 hours, then stop gracefully
+    endtime = time.time() + 12 * 60 * 60
+    max_build_time_reached = False
     for appid, app in apps.items():
-        max_apps_per_run -= 1
-        if max_apps_per_run < 1:
-            break
 
         first = True
 
         for build in app.builds:
+            if time.time() > endtime:
+                max_build_time_reached = True
+                break
+            if options.server:  # enable watchdog timer
+                timer = threading.Timer(7200, force_halt_build)
+                timer.start()
+            else:
+                timer = None
+
             wikilog = None
             build_starttime = common.get_wiki_timestamp()
             tools_version_log = ''
@@ -1287,6 +1308,13 @@ def main():
                 except Exception as e:
                     logging.error("Error while attempting to publish build log: %s" % e)
 
+            if timer:
+                timer.cancel()  # kill the watchdog timer
+
+        if max_build_time_reached:
+            logging.info("Stopping after global build timeout...")
+            break
+
     for app in build_succeeded:
         logging.info("success: %s" % (app.id))
 

fdroidserver/vmtools.py

@@ -29,6 +29,9 @@ from .common import FDroidException
 from logging import getLogger
 
 from fdroidserver import _
+import threading
+
+lock = threading.Lock()
 
 logger = getLogger('fdroidserver-vmtools')
 
@@ -175,7 +178,6 @@ class FDroidBuildVm():
     This is intended to be a hypervisor independant, fault tolerant
     wrapper around the vagrant functions we use.
     """
-
     def __init__(self, srvdir):
         """Create new server class.
         """
@@ -191,21 +193,27 @@ class FDroidBuildVm():
         self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
 
     def up(self, provision=True):
-        try:
+        global lock
-            self.vgrnt.up(provision=provision)
+        with lock:
-            self.srvuuid = self._vagrant_fetch_uuid()
+            try:
-        except subprocess.CalledProcessError as e:
+                self.vgrnt.up(provision=provision)
-            raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e
+                self.srvuuid = self._vagrant_fetch_uuid()
+            except subprocess.CalledProcessError as e:
+                raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e
 
     def suspend(self):
-        logger.info('suspending buildserver')
+        global lock
-        try:
+        with lock:
-            self.vgrnt.suspend()
+            logger.info('suspending buildserver')
-        except subprocess.CalledProcessError as e:
+            try:
-            raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e
+                self.vgrnt.suspend()
+            except subprocess.CalledProcessError as e:
+                raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e
 
     def halt(self):
-        self.vgrnt.halt(force=True)
+        global lock
+        with lock:
+            self.vgrnt.halt(force=True)
 
     def destroy(self):
         """Remove every trace of this VM from the system.