mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-07-07 09:50:07 +02:00
buildserver: move code into main() method to always stop thread
By running the whole program in a main() function, it can be wrapped in try/finally in order to stop the background display thread. This is also done in ./fdroid, its standard practice for Python CLI utilities.
This commit is contained in:
parent
675500ad88
commit
6464ec55b7
295
makebuildserver
295
makebuildserver
@ -20,7 +20,7 @@ if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
|
|||||||
|
|
||||||
|
|
||||||
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
|
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
|
||||||
serverdir = 'buildserver'
|
tail = None
|
||||||
|
|
||||||
parser = OptionParser()
|
parser = OptionParser()
|
||||||
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
||||||
@ -293,158 +293,167 @@ def sha256_for_file(path):
|
|||||||
return s.hexdigest()
|
return s.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
for srcurl, shasum in cachefiles:
|
def main():
|
||||||
filename = os.path.basename(srcurl)
|
global cachedir, cachefiles, config, tail
|
||||||
local_filename = os.path.join(cachedir, filename)
|
|
||||||
|
|
||||||
if os.path.exists(local_filename):
|
for srcurl, shasum in cachefiles:
|
||||||
local_length = os.path.getsize(local_filename)
|
filename = os.path.basename(srcurl)
|
||||||
else:
|
local_filename = os.path.join(cachedir, filename)
|
||||||
local_length = -1
|
|
||||||
|
|
||||||
resume_header = {}
|
if os.path.exists(local_filename):
|
||||||
download = True
|
local_length = os.path.getsize(local_filename)
|
||||||
|
|
||||||
try:
|
|
||||||
r = requests.head(srcurl, allow_redirects=True, timeout=60)
|
|
||||||
if r.status_code == 200:
|
|
||||||
content_length = int(r.headers.get('content-length'))
|
|
||||||
else:
|
else:
|
||||||
|
local_length = -1
|
||||||
|
|
||||||
|
resume_header = {}
|
||||||
|
download = True
|
||||||
|
|
||||||
|
try:
|
||||||
|
r = requests.head(srcurl, allow_redirects=True, timeout=60)
|
||||||
|
if r.status_code == 200:
|
||||||
|
content_length = int(r.headers.get('content-length'))
|
||||||
|
else:
|
||||||
|
content_length = local_length # skip the download
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
content_length = local_length # skip the download
|
content_length = local_length # skip the download
|
||||||
except requests.exceptions.RequestException as e:
|
print(e)
|
||||||
content_length = local_length # skip the download
|
|
||||||
print(e)
|
|
||||||
|
|
||||||
if local_length == content_length:
|
if local_length == content_length:
|
||||||
download = False
|
download = False
|
||||||
elif local_length > content_length:
|
elif local_length > content_length:
|
||||||
print('deleting corrupt file from cache: ' + local_filename)
|
print('deleting corrupt file from cache: ' + local_filename)
|
||||||
os.remove(local_filename)
|
os.remove(local_filename)
|
||||||
print("Downloading " + filename + " to cache")
|
print("Downloading " + filename + " to cache")
|
||||||
elif local_length > -1 and local_length < content_length:
|
elif local_length > -1 and local_length < content_length:
|
||||||
print("Resuming download of " + local_filename)
|
print("Resuming download of " + local_filename)
|
||||||
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
|
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
|
||||||
else:
|
|
||||||
print("Downloading " + filename + " to cache")
|
|
||||||
|
|
||||||
if download:
|
|
||||||
r = requests.get(srcurl, headers=resume_header,
|
|
||||||
stream=True, verify=False, allow_redirects=True)
|
|
||||||
content_length = int(r.headers.get('content-length'))
|
|
||||||
with open(local_filename, 'ab') as f:
|
|
||||||
for chunk in progress.bar(r.iter_content(chunk_size=65536),
|
|
||||||
expected_size=(content_length / 65536) + 1):
|
|
||||||
if chunk: # filter out keep-alive new chunks
|
|
||||||
f.write(chunk)
|
|
||||||
|
|
||||||
v = sha256_for_file(local_filename)
|
|
||||||
if v == shasum:
|
|
||||||
print("\t...shasum verified for " + local_filename)
|
|
||||||
else:
|
|
||||||
print("Invalid shasum of '" + v + "' detected for " + local_filename)
|
|
||||||
os.remove(local_filename)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
|
|
||||||
print("Setting executable bit for " + local_qt_filename)
|
|
||||||
os.chmod(local_qt_filename, 0o755)
|
|
||||||
|
|
||||||
# use VirtualBox software virtualization if hardware is not available,
|
|
||||||
# like if this is being run in kvm or some other VM platform, like
|
|
||||||
# http://jenkins.debian.net, the values are 'on' or 'off'
|
|
||||||
if sys.platform.startswith('darwin'):
|
|
||||||
# all < 10 year old Macs work, and OSX servers as VM host are very
|
|
||||||
# rare, but this could also be auto-detected if someone codes it
|
|
||||||
config['hwvirtex'] = 'on'
|
|
||||||
elif os.path.exists('/proc/cpuinfo'):
|
|
||||||
with open('/proc/cpuinfo') as f:
|
|
||||||
contents = f.read()
|
|
||||||
if 'vmx' in contents or 'svm' in contents:
|
|
||||||
config['hwvirtex'] = 'on'
|
|
||||||
|
|
||||||
logfilename = os.path.join(serverdir, 'up.log')
|
|
||||||
if not os.path.exists(logfilename):
|
|
||||||
open(logfilename, 'a').close() # create blank file
|
|
||||||
log_cm = vagrant.make_file_cm(logfilename)
|
|
||||||
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
|
|
||||||
|
|
||||||
if options.verbose:
|
|
||||||
tail = fdroidserver.tail.Tail(logfilename)
|
|
||||||
tail.start()
|
|
||||||
|
|
||||||
if options.clean:
|
|
||||||
v.destroy()
|
|
||||||
if config['vm_provider'] == 'libvirt':
|
|
||||||
subprocess.call(['virsh', 'undefine', 'buildserver_default'])
|
|
||||||
subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img'])
|
|
||||||
|
|
||||||
# Check against the existing Vagrantfile.yaml, and if they differ, we
|
|
||||||
# need to create a new box:
|
|
||||||
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
|
||||||
writevf = True
|
|
||||||
if os.path.exists(vf):
|
|
||||||
print('Halting', serverdir)
|
|
||||||
v.halt()
|
|
||||||
with open(vf, 'r', encoding='utf-8') as f:
|
|
||||||
oldconfig = yaml.load(f)
|
|
||||||
if config != oldconfig:
|
|
||||||
print("Server configuration has changed, rebuild from scratch is required")
|
|
||||||
v.destroy()
|
|
||||||
else:
|
|
||||||
print("Re-provisioning existing server")
|
|
||||||
writevf = False
|
|
||||||
else:
|
|
||||||
print("No existing server - building from scratch")
|
|
||||||
if writevf:
|
|
||||||
with open(vf, 'w', encoding='utf-8') as f:
|
|
||||||
yaml.dump(config, f)
|
|
||||||
|
|
||||||
if config['vm_provider'] == 'libvirt':
|
|
||||||
found_basebox = False
|
|
||||||
needs_mutate = False
|
|
||||||
for box in v.box_list():
|
|
||||||
if box.name == config['basebox']:
|
|
||||||
found_basebox = True
|
|
||||||
if box.provider != 'libvirt':
|
|
||||||
needs_mutate = True
|
|
||||||
continue
|
|
||||||
if not found_basebox:
|
|
||||||
if isinstance(config['baseboxurl'], str):
|
|
||||||
baseboxurl = config['baseboxurl']
|
|
||||||
else:
|
else:
|
||||||
baseboxurl = config['baseboxurl'][0]
|
print("Downloading " + filename + " to cache")
|
||||||
print('Adding', config['basebox'], 'from', baseboxurl)
|
|
||||||
v.box_add(config['basebox'], baseboxurl)
|
|
||||||
needs_mutate = True
|
|
||||||
if needs_mutate:
|
|
||||||
print('Converting', config['basebox'], 'to libvirt format')
|
|
||||||
v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
|
|
||||||
print('Removing virtualbox format copy of', config['basebox'])
|
|
||||||
v.box_remove(config['basebox'], 'virtualbox')
|
|
||||||
|
|
||||||
print("Configuring build server VM")
|
if download:
|
||||||
v.up(provision=True)
|
r = requests.get(srcurl, headers=resume_header,
|
||||||
|
stream=True, verify=False, allow_redirects=True)
|
||||||
|
content_length = int(r.headers.get('content-length'))
|
||||||
|
with open(local_filename, 'ab') as f:
|
||||||
|
for chunk in progress.bar(r.iter_content(chunk_size=65536),
|
||||||
|
expected_size=(content_length / 65536) + 1):
|
||||||
|
if chunk: # filter out keep-alive new chunks
|
||||||
|
f.write(chunk)
|
||||||
|
|
||||||
print("Writing buildserver ID")
|
v = sha256_for_file(local_filename)
|
||||||
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
if v == shasum:
|
||||||
universal_newlines=True)
|
print("\t...shasum verified for " + local_filename)
|
||||||
buildserverid = p.communicate()[0].strip()
|
else:
|
||||||
print("...ID is " + buildserverid)
|
print("Invalid shasum of '" + v + "' detected for " + local_filename)
|
||||||
subprocess.call(
|
os.remove(local_filename)
|
||||||
['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"'
|
sys.exit(1)
|
||||||
.format(buildserverid)],
|
|
||||||
cwd=serverdir)
|
|
||||||
|
|
||||||
print("Stopping build server VM")
|
local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
|
||||||
v.halt()
|
print("Setting executable bit for " + local_qt_filename)
|
||||||
|
os.chmod(local_qt_filename, 0o755)
|
||||||
|
|
||||||
print("Packaging")
|
# use VirtualBox software virtualization if hardware is not available,
|
||||||
v.package(output=boxfile)
|
# like if this is being run in kvm or some other VM platform, like
|
||||||
|
# http://jenkins.debian.net, the values are 'on' or 'off'
|
||||||
|
if sys.platform.startswith('darwin'):
|
||||||
|
# all < 10 year old Macs work, and OSX servers as VM host are very
|
||||||
|
# rare, but this could also be auto-detected if someone codes it
|
||||||
|
config['hwvirtex'] = 'on'
|
||||||
|
elif os.path.exists('/proc/cpuinfo'):
|
||||||
|
with open('/proc/cpuinfo') as f:
|
||||||
|
contents = f.read()
|
||||||
|
if 'vmx' in contents or 'svm' in contents:
|
||||||
|
config['hwvirtex'] = 'on'
|
||||||
|
|
||||||
print("Adding box")
|
serverdir = os.path.join(os.getcwd(), 'buildserver')
|
||||||
v.box_add('buildserver', boxfile, force=True)
|
logfilename = os.path.join(serverdir, 'up.log')
|
||||||
|
if not os.path.exists(logfilename):
|
||||||
|
open(logfilename, 'a').close() # create blank file
|
||||||
|
log_cm = vagrant.make_file_cm(logfilename)
|
||||||
|
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
|
||||||
|
|
||||||
os.remove(boxfile)
|
if options.verbose:
|
||||||
|
tail = fdroidserver.tail.Tail(logfilename)
|
||||||
|
tail.start()
|
||||||
|
|
||||||
if tail is not None:
|
if options.clean:
|
||||||
tail.stop()
|
v.destroy()
|
||||||
|
if config['vm_provider'] == 'libvirt':
|
||||||
|
subprocess.call(['virsh', 'undefine', 'buildserver_default'])
|
||||||
|
subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img'])
|
||||||
|
|
||||||
|
# Check against the existing Vagrantfile.yaml, and if they differ, we
|
||||||
|
# need to create a new box:
|
||||||
|
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
||||||
|
writevf = True
|
||||||
|
if os.path.exists(vf):
|
||||||
|
print('Halting', serverdir)
|
||||||
|
v.halt()
|
||||||
|
with open(vf, 'r', encoding='utf-8') as f:
|
||||||
|
oldconfig = yaml.load(f)
|
||||||
|
if config != oldconfig:
|
||||||
|
print("Server configuration has changed, rebuild from scratch is required")
|
||||||
|
v.destroy()
|
||||||
|
else:
|
||||||
|
print("Re-provisioning existing server")
|
||||||
|
writevf = False
|
||||||
|
else:
|
||||||
|
print("No existing server - building from scratch")
|
||||||
|
if writevf:
|
||||||
|
with open(vf, 'w', encoding='utf-8') as f:
|
||||||
|
yaml.dump(config, f)
|
||||||
|
|
||||||
|
if config['vm_provider'] == 'libvirt':
|
||||||
|
found_basebox = False
|
||||||
|
needs_mutate = False
|
||||||
|
for box in v.box_list():
|
||||||
|
if box.name == config['basebox']:
|
||||||
|
found_basebox = True
|
||||||
|
if box.provider != 'libvirt':
|
||||||
|
needs_mutate = True
|
||||||
|
continue
|
||||||
|
if not found_basebox:
|
||||||
|
if isinstance(config['baseboxurl'], str):
|
||||||
|
baseboxurl = config['baseboxurl']
|
||||||
|
else:
|
||||||
|
baseboxurl = config['baseboxurl'][0]
|
||||||
|
print('Adding', config['basebox'], 'from', baseboxurl)
|
||||||
|
v.box_add(config['basebox'], baseboxurl)
|
||||||
|
needs_mutate = True
|
||||||
|
if needs_mutate:
|
||||||
|
print('Converting', config['basebox'], 'to libvirt format')
|
||||||
|
v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
|
||||||
|
print('Removing virtualbox format copy of', config['basebox'])
|
||||||
|
v.box_remove(config['basebox'], 'virtualbox')
|
||||||
|
|
||||||
|
print("Configuring build server VM")
|
||||||
|
v.up(provision=True)
|
||||||
|
|
||||||
|
print("Writing buildserver ID")
|
||||||
|
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
||||||
|
universal_newlines=True)
|
||||||
|
buildserverid = p.communicate()[0].strip()
|
||||||
|
print("...ID is " + buildserverid)
|
||||||
|
subprocess.call(
|
||||||
|
['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"'
|
||||||
|
.format(buildserverid)],
|
||||||
|
cwd=serverdir)
|
||||||
|
|
||||||
|
print("Stopping build server VM")
|
||||||
|
v.halt()
|
||||||
|
|
||||||
|
print("Packaging")
|
||||||
|
v.package(output=boxfile)
|
||||||
|
|
||||||
|
print("Adding box")
|
||||||
|
v.box_add('buildserver', boxfile, force=True)
|
||||||
|
|
||||||
|
os.remove(boxfile)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
finally:
|
||||||
|
if tail is not None:
|
||||||
|
tail.stop()
|
||||||
|
Loading…
Reference in New Issue
Block a user