mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-13 02:30:11 +01:00
Buildserver snapshot refinements
This commit is contained in:
Ciaran Gultnieks
parent
448fa61d0c
commit
cdfa020ba7
@ -49,7 +49,12 @@ def build_server(app, thisbuild, build_dir, output_dir):
|
||||
output = p.communicate()[0]
|
||||
if output.find('fdroidclean') != -1:
|
||||
print "...snapshot exists - resetting build server to clean state"
|
||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||
p = subprocess.Popen(['vagrant', 'status'],
|
||||
cwd='builder', stdout=subprocess.PIPE)
|
||||
output = p.communicate()[0]
|
||||
if output.find('running') != -1:
|
||||
print "...suspending"
|
||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||
if subprocess.call(['vagrant', 'snap', 'go', 'fdroidclean'],
|
||||
cwd='builder') == 0:
|
||||
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
|
||||
@ -89,122 +94,129 @@ def build_server(app, thisbuild, build_dir, output_dir):
|
||||
if output.find('fdroidclean') == -1:
|
||||
raise BuildException("Failed to take snapshot.")
|
||||
|
||||
# Get SSH configuration settings for us to connect...
|
||||
subprocess.call('vagrant ssh-config >sshconfig',
|
||||
cwd='builder', shell=True)
|
||||
vagranthost = 'default' # Host in ssh config file
|
||||
|
||||
# Load and parse the SSH config...
|
||||
sshconfig = ssh.SSHConfig()
|
||||
sshf = open('builder/sshconfig', 'r')
|
||||
sshconfig.parse(sshf)
|
||||
sshf.close()
|
||||
sshconfig = sshconfig.lookup(vagranthost)
|
||||
|
||||
# Open SSH connection...
|
||||
sshs = ssh.SSHClient()
|
||||
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
|
||||
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
|
||||
port=int(sshconfig['port']), timeout=10, look_for_keys=False,
|
||||
key_filename=sshconfig['identityfile'])
|
||||
|
||||
# Get an SFTP connection...
|
||||
ftp = sshs.open_sftp()
|
||||
ftp.get_channel().settimeout(15)
|
||||
|
||||
# Put all the necessary files in place...
|
||||
ftp.chdir('/home/vagrant')
|
||||
|
||||
# Helper to copy the contents of a directory to the server...
|
||||
def send_dir(path):
|
||||
root = os.path.dirname(path)
|
||||
main = os.path.basename(path)
|
||||
ftp.mkdir(main)
|
||||
for r, d, f in os.walk(path):
|
||||
rr = os.path.relpath(r, root)
|
||||
ftp.chdir(rr)
|
||||
for dd in d:
|
||||
ftp.mkdir(dd)
|
||||
for ff in f:
|
||||
ftp.put(os.path.join(root, rr, ff), ff)
|
||||
for i in range(len(rr.split('/'))):
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
|
||||
print "Preparing server for build..."
|
||||
serverpath = os.path.abspath(os.path.dirname(__file__))
|
||||
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
|
||||
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
|
||||
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py')
|
||||
|
||||
# Copy the metadata - just the file for this app...
|
||||
ftp.mkdir('metadata')
|
||||
ftp.chdir('metadata')
|
||||
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
|
||||
app['id'] + '.txt')
|
||||
ftp.chdir('..')
|
||||
# Create the build directory...
|
||||
ftp.mkdir('build')
|
||||
ftp.chdir('build')
|
||||
ftp.mkdir('extlib')
|
||||
# Copy the main app source code
|
||||
if os.path.exists(build_dir):
|
||||
send_dir(build_dir)
|
||||
# Copy any extlibs that are required...
|
||||
if thisbuild.has_key('extlibs'):
|
||||
ftp.chdir('build')
|
||||
ftp.chdir('extlib')
|
||||
for lib in thisbuild['extlibs'].split(';'):
|
||||
lp = lib.split('/')
|
||||
for d in lp[:-1]:
|
||||
ftp.mkdir(d)
|
||||
ftp.chdir(d)
|
||||
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
||||
for _ in lp[:-1]:
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
# Copy any srclibs that are required...
|
||||
if thisbuild.has_key('srclibs'):
|
||||
ftp.chdir('build')
|
||||
ftp.chdir('extlib')
|
||||
for lib in thisbuild['srclibs'].split(';'):
|
||||
lp = lib.split('@').split('/')
|
||||
for d in lp[:-1]:
|
||||
ftp.mkdir(d)
|
||||
ftp.chdir(d)
|
||||
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
||||
for _ in lp[:-1]:
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
|
||||
# Execute the build script...
|
||||
print "Starting build..."
|
||||
chan = sshs.get_transport().open_session()
|
||||
stdoutf = chan.makefile('rb')
|
||||
stderrf = chan.makefile_stderr('rb')
|
||||
chan.exec_command('python build.py --on-server -p ' +
|
||||
app['id'] + ' --vercode ' + thisbuild['vercode'])
|
||||
returncode = chan.recv_exit_status()
|
||||
output = stdoutf.read()
|
||||
error = stderrf.read()
|
||||
if returncode != 0:
|
||||
raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
||||
|
||||
# Retrieve the built files...
|
||||
ftp.chdir('/home/vagrant/unsigned')
|
||||
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
|
||||
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
|
||||
try:
|
||||
ftp.get(apkfile, os.path.join(output_dir, apkfile))
|
||||
ftp.get(tarball, os.path.join(output_dir, tarball))
|
||||
except:
|
||||
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
||||
ftp.close()
|
||||
|
||||
# Suspend the buildserver...
|
||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||
# Get SSH configuration settings for us to connect...
|
||||
print "Getting ssh configuration..."
|
||||
subprocess.call('vagrant ssh-config >sshconfig',
|
||||
cwd='builder', shell=True)
|
||||
vagranthost = 'default' # Host in ssh config file
|
||||
|
||||
# Load and parse the SSH config...
|
||||
sshconfig = ssh.SSHConfig()
|
||||
sshf = open('builder/sshconfig', 'r')
|
||||
sshconfig.parse(sshf)
|
||||
sshf.close()
|
||||
sshconfig = sshconfig.lookup(vagranthost)
|
||||
|
||||
# Open SSH connection...
|
||||
print "Connecting to virtual machine..."
|
||||
sshs = ssh.SSHClient()
|
||||
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
|
||||
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
|
||||
port=int(sshconfig['port']), timeout=10, look_for_keys=False,
|
||||
key_filename=sshconfig['identityfile'])
|
||||
|
||||
# Get an SFTP connection...
|
||||
ftp = sshs.open_sftp()
|
||||
ftp.get_channel().settimeout(15)
|
||||
|
||||
# Put all the necessary files in place...
|
||||
ftp.chdir('/home/vagrant')
|
||||
|
||||
# Helper to copy the contents of a directory to the server...
|
||||
def send_dir(path):
|
||||
root = os.path.dirname(path)
|
||||
main = os.path.basename(path)
|
||||
ftp.mkdir(main)
|
||||
for r, d, f in os.walk(path):
|
||||
rr = os.path.relpath(r, root)
|
||||
ftp.chdir(rr)
|
||||
for dd in d:
|
||||
ftp.mkdir(dd)
|
||||
for ff in f:
|
||||
if not os.path.islink(os.path.join(root, rr, ff)):
|
||||
ftp.put(os.path.join(root, rr, ff), ff)
|
||||
for i in range(len(rr.split('/'))):
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
|
||||
print "Preparing server for build..."
|
||||
serverpath = os.path.abspath(os.path.dirname(__file__))
|
||||
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
|
||||
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
|
||||
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py')
|
||||
|
||||
# Copy the metadata - just the file for this app...
|
||||
ftp.mkdir('metadata')
|
||||
ftp.chdir('metadata')
|
||||
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
|
||||
app['id'] + '.txt')
|
||||
ftp.chdir('..')
|
||||
# Create the build directory...
|
||||
ftp.mkdir('build')
|
||||
ftp.chdir('build')
|
||||
ftp.mkdir('extlib')
|
||||
# Copy the main app source code
|
||||
if os.path.exists(build_dir):
|
||||
send_dir(build_dir)
|
||||
# Copy any extlibs that are required...
|
||||
if thisbuild.has_key('extlibs'):
|
||||
ftp.chdir('build')
|
||||
ftp.chdir('extlib')
|
||||
for lib in thisbuild['extlibs'].split(';'):
|
||||
lp = lib.split('/')
|
||||
for d in lp[:-1]:
|
||||
ftp.mkdir(d)
|
||||
ftp.chdir(d)
|
||||
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
||||
for _ in lp[:-1]:
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
# Copy any srclibs that are required...
|
||||
if thisbuild.has_key('srclibs'):
|
||||
ftp.chdir('build')
|
||||
ftp.chdir('extlib')
|
||||
for lib in thisbuild['srclibs'].split(';'):
|
||||
lp = lib.split('@').split('/')
|
||||
for d in lp[:-1]:
|
||||
ftp.mkdir(d)
|
||||
ftp.chdir(d)
|
||||
ftp.put(os.path.join('build/extlib', lib), lp[-1])
|
||||
for _ in lp[:-1]:
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
ftp.chdir('..')
|
||||
|
||||
# Execute the build script...
|
||||
print "Starting build..."
|
||||
chan = sshs.get_transport().open_session()
|
||||
stdoutf = chan.makefile('rb')
|
||||
stderrf = chan.makefile_stderr('rb')
|
||||
chan.exec_command('python build.py --on-server -p ' +
|
||||
app['id'] + ' --vercode ' + thisbuild['vercode'])
|
||||
returncode = chan.recv_exit_status()
|
||||
output = stdoutf.read()
|
||||
error = stderrf.read()
|
||||
if returncode != 0:
|
||||
raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
||||
|
||||
# Retrieve the built files...
|
||||
ftp.chdir('/home/vagrant/unsigned')
|
||||
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
|
||||
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
|
||||
try:
|
||||
ftp.get(apkfile, os.path.join(output_dir, apkfile))
|
||||
ftp.get(tarball, os.path.join(output_dir, tarball))
|
||||
except:
|
||||
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
|
||||
ftp.close()
|
||||
|
||||
finally:
|
||||
|
||||
# Suspend the build server.
|
||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||
|
||||
|
||||
def build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force, verbose=False):
|
||||
|
||||
@ -2,12 +2,9 @@
|
||||
set -e
|
||||
rm -f buildserver.box
|
||||
cd buildserver
|
||||
vagrant halt
|
||||
vagrant up
|
||||
sleep 5
|
||||
vagrant ssh -c "sudo shutdown -h now"
|
||||
cd ..
|
||||
# Just to wait until it's shut down!
|
||||
sleep 20
|
||||
vagrant halt
|
||||
vagrant package --base `VBoxManage list vms | grep buildserver | sed 's/"\(.*\)".*/\1/'` --output buildserver.box
|
||||
vagrant box add buildserver buildserver.box -f && rm buildserver.box
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user