buildserver: replace custom code with python-vagrant

I ran into some annoying issues with UTF-8 output in the vagrant logs, and
it was hard to solve.  So I switched to using python-vagrant, which handles
it all for us.  Its been around since 2012, has a number of contributors,
and is still actively maintained, so it seems like a good bet. I also
packaged it for Debian, including a backport in jessie-backports.

On Debian/jessie, do `apt-get install python3-vagrant/jessie-backports`

makebuildserver

@@ -6,7 +6,7 @@ import requests
 import stat
 import sys
 import subprocess
-import time
+import vagrant
 import hashlib
 import yaml
 from clint.textui import progress
@@ -18,34 +18,7 @@ if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
     sys.exit(1)
 
 
-def vagrant(params, cwd=None, printout=False):
+boxfile = os.path.join(os.getcwd(), 'buildserver.box')
-    """Run vagrant.
-
-    :param: list of parameters to pass to vagrant
-    :cwd: directory to run in, or None for current directory
-    :printout: True to print output in realtime, False to just
-               return it
-    :returns: (ret, out) where ret is the return code, and out
-               is the stdout (and stderr) from vagrant
-    """
-    p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
-                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                         universal_newlines=True)
-    out = ''
-    if printout:
-        while True:
-            line = p.stdout.readline()
-            if len(line) == 0:
-                break
-            print(line.rstrip())
-            out += line
-        p.wait()
-    else:
-        out = p.communicate()[0]
-    return (p.returncode, out)
-
-
-boxfile = 'buildserver.box'
 serverdir = 'buildserver'
 
 parser = OptionParser()
@@ -96,12 +69,6 @@ if '__builtins__' in config:
 if os.path.exists(boxfile):
     os.remove(boxfile)
 
-if options.clean:
-    vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose)
-    if config['vm_provider'] == 'libvirt':
-        subprocess.call(['virsh', 'undefine', 'buildserver_default'])
-        subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img'])
-
 # Update cached files.
 cachedir = config['cachedir']
 if not os.path.exists(cachedir):
@@ -394,18 +361,28 @@ elif os.path.exists('/proc/cpuinfo'):
     if 'vmx' in contents or 'svm' in contents:
         config['hwvirtex'] = 'on'
 
+logfilename = os.path.join(serverdir, 'up.log')
+log_cm = vagrant.make_file_cm(logfilename)
+v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
+
+if options.clean:
+    v.destroy()
+    if config['vm_provider'] == 'libvirt':
+        subprocess.call(['virsh', 'undefine', 'buildserver_default'])
+        subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img'])
+
 # Check against the existing Vagrantfile.yaml, and if they differ, we
 # need to create a new box:
 vf = os.path.join(serverdir, 'Vagrantfile.yaml')
 writevf = True
 if os.path.exists(vf):
     print('Halting', serverdir)
-    vagrant(['halt'], serverdir)
+    v.halt()
     with open(vf, 'r', encoding='utf-8') as f:
         oldconfig = yaml.load(f)
     if config != oldconfig:
         print("Server configuration has changed, rebuild from scratch is required")
-        vagrant(['destroy', '-f'], serverdir)
+        v.destroy()
     else:
         print("Re-provisioning existing server")
         writevf = False
@@ -416,13 +393,12 @@ if writevf:
         yaml.dump(config, f)
 
 if config['vm_provider'] == 'libvirt':
-    returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose)
     found_basebox = False
     needs_mutate = False
-    for line in out.splitlines():
+    for box in v.box_list():
-        if line.startswith(config['basebox']):
+        if box.name == config['basebox']:
             found_basebox = True
-            if line.split('(')[1].split(',')[0] != 'libvirt':
+            if box.provider != 'libvirt':
                 needs_mutate = True
             continue
     if not found_basebox:
@@ -431,24 +407,16 @@ if config['vm_provider'] == 'libvirt':
         else:
             baseboxurl = config['baseboxurl'][0]
         print('Adding', config['basebox'], 'from', baseboxurl)
-        vagrant(['box', 'add', '--name', config['basebox'], baseboxurl],
+        v.box_add(config['basebox'], baseboxurl)
-                serverdir, printout=options.verbose)
         needs_mutate = True
     if needs_mutate:
         print('Converting', config['basebox'], 'to libvirt format')
-        vagrant(['mutate', config['basebox'], 'libvirt'],
+        v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
-                serverdir, printout=options.verbose)
         print('Removing virtualbox format copy of', config['basebox'])
-        vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']],
+        v.box_remove(config['basebox'], 'virtualbox')
-                serverdir, printout=options.verbose)
 
 print("Configuring build server VM")
-returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)
+v.up(provision=True)
-with open(os.path.join(serverdir, 'up.log'), 'w') as log:
-    log.write(out)
-if returncode != 0:
-    print("Failed to configure server")
-    sys.exit(1)
 
 print("Writing buildserver ID")
 p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
@@ -461,28 +429,12 @@ subprocess.call(
     cwd=serverdir)
 
 print("Stopping build server VM")
-vagrant(['halt'], serverdir)
+v.halt()
-
-print("Waiting for build server VM to be finished")
-ready = False
-while not ready:
-    time.sleep(2)
-    returncode, out = vagrant(['status'], serverdir)
-    if returncode != 0:
-        print("Error while checking status")
-        sys.exit(1)
-    for line in out.splitlines():
-        if line.startswith("default"):
-            if line.find("poweroff") != -1 or line.find("shutoff") != 1:
-                ready = True
-            else:
-                print("Status: " + line)
 
 print("Packaging")
-vagrant(['package', '--output', os.path.join('..', boxfile)], serverdir,
+v.package(output=boxfile)
-        printout=options.verbose)
+
 print("Adding box")
-vagrant(['box', 'add', 'buildserver', boxfile, '-f'],
+v.box_add('buildserver', boxfile, force=True)
-        printout=options.verbose)
 
 os.remove(boxfile)

setup.py

@@ -36,6 +36,7 @@ setup(name='fdroidserver',
           'apache-libcloud >= 0.14.1',
           'pyasn1',
           'pyasn1-modules',
+          'python-vagrant',
           'PyYAML',
           'requests < 2.11',
           'docker-py == 1.9.0',