mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-09 00:40:11 +01:00
584 lines
27 KiB
Python
Executable File
584 lines
27 KiB
Python
Executable File
#!/usr/bin/env python2
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# build.py - part of the FDroid server tools
|
|
# Copyright (C) 2010, Ciaran Gultnieks, ciaran@ciarang.com
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import sys
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import re
|
|
import zipfile
|
|
import tarfile
|
|
import md5
|
|
import shlex
|
|
from xml.dom.minidom import Document
|
|
from optparse import OptionParser
|
|
|
|
import common
|
|
|
|
#Read configuration...
|
|
execfile('config.py')
|
|
|
|
|
|
# Parse command line...
|
|
parser = OptionParser()
|
|
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
|
help="Spew out even more information than normal")
|
|
parser.add_option("-p", "--package", default=None,
|
|
help="Build only the specified package")
|
|
parser.add_option("-c", "--clean", action="store_true", default=False,
|
|
help="Clean mode - build everything from scratch")
|
|
(options, args) = parser.parse_args()
|
|
|
|
# Get all apps...
|
|
apps = common.read_metadata(options.verbose)
|
|
|
|
#Clear and/or create the 'built' directory, depending on mode:
|
|
built_dir = 'built'
|
|
if options.clean:
|
|
if os.path.exists(built_dir):
|
|
shutil.rmtree(built_dir)
|
|
if not os.path.exists(built_dir):
|
|
os.mkdir(built_dir)
|
|
|
|
if not os.path.isdir('build'):
|
|
os.makedirs('build')
|
|
|
|
for app in apps:
|
|
|
|
if app['disabled']:
|
|
print "Skipping %s: disabled" % app['id']
|
|
elif not app['builds']:
|
|
print "Skipping %s: no builds specified" % app['id']
|
|
|
|
if (app['disabled'] is None and app['repo'] != ''
|
|
and app['repotype'] != '' and (options.package is None or
|
|
options.package == app['id']) and len(app['builds']) > 0):
|
|
|
|
print "Processing " + app['id']
|
|
|
|
build_dir = 'build/' + app['id']
|
|
|
|
got_source = False
|
|
|
|
|
|
for thisbuild in app['builds']:
|
|
|
|
dest = os.path.join(built_dir, app['id'] + '_' +
|
|
thisbuild['vercode'] + '.apk')
|
|
dest_unsigned = dest + "_unsigned"
|
|
|
|
if os.path.exists(dest):
|
|
print "..version " + thisbuild['version'] + " already exists"
|
|
elif thisbuild['commit'].startswith('!'):
|
|
print ("..skipping version " + thisbuild['version'] + " - " +
|
|
thisbuild['commit'][1:])
|
|
else:
|
|
print "..building version " + thisbuild['version']
|
|
|
|
if not got_source:
|
|
|
|
got_source = True
|
|
|
|
# Remove the build directory if it already exists...
|
|
if os.path.exists(build_dir):
|
|
shutil.rmtree(build_dir)
|
|
|
|
# Strip username/password out of repo address if specified (relevant
|
|
# only for SVN) and store for use later.
|
|
repo = app['repo']
|
|
index = repo.find('@')
|
|
if index != -1:
|
|
username = repo[:index]
|
|
repo = repo[index+1:]
|
|
index = username.find(':')
|
|
if index == -1:
|
|
print "Password required with username"
|
|
sys.exit(1)
|
|
password = username[index+1:]
|
|
username = username[:index]
|
|
repouserargs = ['--username', username,
|
|
'--password', password, '--non-interactive']
|
|
else:
|
|
repouserargs = []
|
|
|
|
# Get the source code...
|
|
if app['repotype'] == 'git':
|
|
if subprocess.call(['git', 'clone', repo, build_dir]) != 0:
|
|
print "Git clone failed"
|
|
sys.exit(1)
|
|
elif app['repotype'] == 'svn':
|
|
if not repo.endswith("*"):
|
|
if subprocess.call(['svn', 'checkout', repo, build_dir] +
|
|
repouserargs) != 0:
|
|
print "Svn checkout failed"
|
|
sys.exit(1)
|
|
elif app['repotype'] == 'hg':
|
|
if subprocess.call(['hg', 'clone', repo, build_dir]) !=0:
|
|
print "Hg clone failed"
|
|
sys.exit(1)
|
|
elif app['repotype'] == 'bzr':
|
|
if subprocess.call(['bzr', 'branch', repo, build_dir]) !=0:
|
|
print "Bzr branch failed"
|
|
sys.exit(1)
|
|
else:
|
|
print "Invalid repo type " + app['repotype'] + " in " + app['id']
|
|
sys.exit(1)
|
|
|
|
# Optionally, the actual app source can be in a subdirectory...
|
|
doupdate = True
|
|
if thisbuild.has_key('subdir'):
|
|
if app['repotype'] == 'svn' and repo.endswith("*"):
|
|
root_dir = build_dir
|
|
# Remove the build directory if it already exists...
|
|
if os.path.exists(build_dir):
|
|
shutil.rmtree(build_dir)
|
|
if subprocess.call(['svn', 'checkout',
|
|
repo[:-1] + thisbuild['subdir'],
|
|
'-r', thisbuild['commit'],
|
|
build_dir] + repouserargs) != 0:
|
|
print "Svn checkout failed"
|
|
sys.exit(1)
|
|
# Because we're checking out for every version we build,
|
|
# we've already checked out the repo at the correct revision
|
|
# and don't need to update to it...
|
|
doupdate = False
|
|
else:
|
|
root_dir = os.path.join(build_dir, thisbuild['subdir'])
|
|
else:
|
|
root_dir = build_dir
|
|
|
|
if doupdate:
|
|
if app['repotype'] == 'git':
|
|
if subprocess.call(['git', 'reset', '--hard', thisbuild['commit']],
|
|
cwd=build_dir) != 0:
|
|
print "Git reset failed"
|
|
sys.exit(1)
|
|
if subprocess.call(['git', 'clean', '-dfx'],
|
|
cwd=build_dir) != 0:
|
|
print "Git clean failed"
|
|
sys.exit(1)
|
|
elif app['repotype'] == 'svn':
|
|
for svncommand in (
|
|
'svn revert -R .',
|
|
r"svn status | awk '/\?/ {print $2}' | xargs rm -rf",
|
|
'svn update --force -r '+thisbuild['commit'],):
|
|
if subprocess.call(svncommand, cwd=build_dir,
|
|
shell=True) != 0:
|
|
print "Svn update failed"
|
|
sys.exit(1)
|
|
elif app['repotype'] == 'hg':
|
|
if subprocess.call(['hg', 'checkout', thisbuild['commit']],
|
|
cwd=build_dir) != 0:
|
|
print "Hg checkout failed"
|
|
sys.exit(1)
|
|
if subprocess.call('hg status -u -0 | xargs rm -rf',
|
|
cwd=build_dir, shell=True) != 0:
|
|
print "Hg clean failed"
|
|
sys.exit(1)
|
|
elif app['repotype'] == 'bzr':
|
|
if subprocess.call(['bzr', 'revert', '-r', thisbuild['commit']],
|
|
cwd=build_dir) != 0:
|
|
print "Bzr revert failed"
|
|
sys.exit(1)
|
|
if subprocess.call(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'],
|
|
cwd=build_dir) != 0:
|
|
print "Bzr revert failed"
|
|
sys.exit(1)
|
|
else:
|
|
print "Invalid repo type " + app['repotype']
|
|
sys.exit(1)
|
|
|
|
# Initialise submodules if requred...
|
|
if thisbuild.get('submodules', 'no') == 'yes':
|
|
if subprocess.call(['git', 'submodule', 'init'],
|
|
cwd=build_dir) != 0:
|
|
print "Git submodule init failed"
|
|
sys.exit(1)
|
|
if subprocess.call(['git', 'submodule', 'update'],
|
|
cwd=build_dir) != 0:
|
|
print "Git submodule update failed"
|
|
sys.exit(1)
|
|
|
|
# Generate (or update) the ant build file, build.xml...
|
|
if (thisbuild.get('update', 'yes') == 'yes' and
|
|
not thisbuild.has_key('maven')):
|
|
parms = [os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p', '.']
|
|
parms.append('--subprojects')
|
|
if thisbuild.has_key('target'):
|
|
parms.append('-t')
|
|
parms.append(thisbuild['target'])
|
|
if subprocess.call(parms, cwd=root_dir) != 0:
|
|
print "Failed to update project"
|
|
sys.exit(1)
|
|
|
|
# If the app has ant set up to sign the release, we need to switch
|
|
# that off, because we want the unsigned apk...
|
|
for propfile in ('build.properties', 'default.properties'):
|
|
if os.path.exists(os.path.join(root_dir, propfile)):
|
|
if subprocess.call(['sed','-i','s/^key.store/#/',
|
|
propfile], cwd=root_dir) !=0:
|
|
print "Failed to amend %s" % propfile
|
|
sys.exit(1)
|
|
|
|
# Update the local.properties file...
|
|
locprops = os.path.join(root_dir, 'local.properties')
|
|
if os.path.exists(locprops):
|
|
f = open(locprops, 'r')
|
|
props = f.read()
|
|
f.close()
|
|
# Fix old-fashioned 'sdk-location' by copying
|
|
# from sdk.dir, if necessary...
|
|
if thisbuild.get('oldsdkloc', 'no') == "yes":
|
|
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
|
|
re.S|re.M).group(1)
|
|
props += "\nsdk-location=" + sdkloc + "\n"
|
|
# Add ndk location...
|
|
props+= "\nndk.dir=" + ndk_path + "\n"
|
|
# Add java.encoding if necessary...
|
|
if thisbuild.has_key('encoding'):
|
|
props += "\njava.encoding=" + thisbuild['encoding'] + "\n"
|
|
f = open(locprops, 'w')
|
|
f.write(props)
|
|
f.close()
|
|
|
|
# Insert version code and number into the manifest if necessary...
|
|
if thisbuild.has_key('insertversion'):
|
|
if subprocess.call(['sed','-i','s/' + thisbuild['insertversion'] +
|
|
'/' + thisbuild['version'] +'/g',
|
|
'AndroidManifest.xml'], cwd=root_dir) !=0:
|
|
print "Failed to amend manifest"
|
|
sys.exit(1)
|
|
if thisbuild.has_key('insertvercode'):
|
|
if subprocess.call(['sed','-i','s/' + thisbuild['insertvercode'] +
|
|
'/' + thisbuild['vercode'] +'/g',
|
|
'AndroidManifest.xml'], cwd=root_dir) !=0:
|
|
print "Failed to amend manifest"
|
|
sys.exit(1)
|
|
|
|
# Delete unwanted file...
|
|
if thisbuild.has_key('rm'):
|
|
os.remove(os.path.join(build_dir, thisbuild['rm']))
|
|
|
|
# Fix apostrophes translation files if necessary...
|
|
if thisbuild.get('fixapos', 'no') == 'yes':
|
|
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
|
|
for filename in files:
|
|
if filename.endswith('.xml'):
|
|
if subprocess.call(['sed','-i','s@' +
|
|
r"\([^\\]\)'@\1\\'" +
|
|
'@g',
|
|
os.path.join(root, filename)]) != 0:
|
|
print "Failed to amend " + filename
|
|
sys.exit(1)
|
|
|
|
# Fix translation files if necessary...
|
|
if thisbuild.get('fixtrans', 'no') == 'yes':
|
|
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
|
|
for filename in files:
|
|
if filename.endswith('.xml'):
|
|
f = open(os.path.join(root, filename))
|
|
changed = False
|
|
outlines = []
|
|
for line in f:
|
|
num = 1
|
|
index = 0
|
|
oldline = line
|
|
while True:
|
|
index = line.find("%", index)
|
|
if index == -1:
|
|
break
|
|
next = line[index+1:index+2]
|
|
if next == "s" or next == "d":
|
|
line = (line[:index+1] +
|
|
str(num) + "$" +
|
|
line[index+1:])
|
|
num += 1
|
|
index += 3
|
|
else:
|
|
index += 1
|
|
# We only want to insert the positional arguments
|
|
# when there is more than one argument...
|
|
if oldline != line:
|
|
if num > 2:
|
|
changed = True
|
|
else:
|
|
line = oldline
|
|
outlines.append(line)
|
|
f.close()
|
|
if changed:
|
|
f = open(os.path.join(root, filename), 'w')
|
|
f.writelines(outlines)
|
|
f.close()
|
|
|
|
# Run a pre-build command if one is required...
|
|
if thisbuild.has_key('prebuild'):
|
|
if subprocess.call(thisbuild['prebuild'],
|
|
cwd=root_dir, shell=True) != 0:
|
|
print "Error running pre-build command"
|
|
sys.exit(1)
|
|
|
|
# Special case init functions for funambol...
|
|
if thisbuild.get('initfun', 'no') == "yes":
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'<taskdef resource="net/sf/antcontrib/antcontrib.properties" />' +
|
|
'@' +
|
|
'<taskdef resource="net/sf/antcontrib/antcontrib.properties">' +
|
|
'<classpath>' +
|
|
'<pathelement location="/usr/share/java/ant-contrib.jar"/>' +
|
|
'</classpath>' +
|
|
'</taskdef>' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
print "Failed to amend build.xml"
|
|
sys.exit(1)
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'\${user.home}/funambol/build/android/build.properties' +
|
|
'@' +
|
|
'build.properties' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
print "Failed to amend build.xml"
|
|
sys.exit(1)
|
|
|
|
buildxml = os.path.join(root_dir, 'build.xml')
|
|
f = open(buildxml, 'r')
|
|
xml = f.read()
|
|
f.close()
|
|
xmlout = ""
|
|
mode = 0
|
|
for line in xml.splitlines():
|
|
if mode == 0:
|
|
if line.find("jarsigner") != -1:
|
|
mode = 1
|
|
else:
|
|
xmlout += line + "\n"
|
|
else:
|
|
if line.find("/exec") != -1:
|
|
mode += 1
|
|
if mode == 3:
|
|
mode =0
|
|
f = open(buildxml, 'w')
|
|
f.write(xmlout)
|
|
f.close()
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'platforms/android-2.0' +
|
|
'@' +
|
|
'platforms/android-8' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
print "Failed to amend build.xml"
|
|
sys.exit(1)
|
|
|
|
shutil.copyfile(
|
|
os.path.join(root_dir, "build.properties.example"),
|
|
os.path.join(root_dir, "build.properties"))
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'javacchome=.*'+
|
|
'@' +
|
|
'javacchome=' + javacc_path +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
print "Failed to amend build.properties"
|
|
sys.exit(1)
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'sdk-folder=.*'+
|
|
'@' +
|
|
'sdk-folder=' + sdk_path +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
print "Failed to amend build.properties"
|
|
sys.exit(1)
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'android.sdk.version.*'+
|
|
'@' +
|
|
'android.sdk.version=2.0' +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
print "Failed to amend build.properties"
|
|
sys.exit(1)
|
|
|
|
|
|
# Build the source tarball right before we build the release...
|
|
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
|
|
tarball = tarfile.open(os.path.join(built_dir,
|
|
tarname + '.tar.gz'), "w:gz")
|
|
tarball.add(build_dir, tarname)
|
|
tarball.close()
|
|
|
|
# Build native stuff if required...
|
|
if thisbuild.get('buildjni', 'no') == 'yes':
|
|
ndkbuild = os.path.join(ndk_path, "ndk-build")
|
|
p = subprocess.Popen([ndkbuild], cwd=root_dir,
|
|
stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
if p.returncode != 0:
|
|
print output
|
|
print "NDK build failed for %s:%s" % (app['id'], thisbuild['version'])
|
|
sys.exit(1)
|
|
elif options.verbose:
|
|
print output
|
|
|
|
# Build the release...
|
|
if thisbuild.has_key('maven'):
|
|
p = subprocess.Popen(['mvn', 'clean', 'install',
|
|
'-Dandroid.sdk.path=' + sdk_path],
|
|
cwd=root_dir, stdout=subprocess.PIPE)
|
|
else:
|
|
if thisbuild.has_key('antcommand'):
|
|
antcommand = thisbuild['antcommand']
|
|
else:
|
|
antcommand = 'release'
|
|
p = subprocess.Popen(['ant', antcommand], cwd=root_dir,
|
|
stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
if p.returncode != 0:
|
|
print output
|
|
print "Build failed for %s:%s" % (app['id'], thisbuild['version'])
|
|
sys.exit(1)
|
|
elif options.verbose:
|
|
print output
|
|
print "Build successful"
|
|
|
|
# Find the apk name in the output...
|
|
if thisbuild.has_key('bindir'):
|
|
bindir = os.path.join(build_dir, thisbuild['bindir'])
|
|
else:
|
|
bindir = os.path.join(root_dir, 'bin')
|
|
if thisbuild.get('initfun', 'no') == "yes":
|
|
# Special case (again!) for funambol...
|
|
src = ("funambol-android-sync-client-" +
|
|
thisbuild['version'] + "-unsigned.apk")
|
|
src = os.path.join(bindir, src)
|
|
elif thisbuild.has_key('maven'):
|
|
src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk",
|
|
output, re.S|re.M).group(1)
|
|
src = os.path.join(bindir, src) + '.apk'
|
|
#[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk
|
|
else:
|
|
src = re.match(r".*^.*Creating (\S+) for release.*$.*", output,
|
|
re.S|re.M).group(1)
|
|
src = os.path.join(bindir, src)
|
|
|
|
# By way of a sanity check, make sure the version and version
|
|
# code in our new apk match what we expect...
|
|
print "Checking " + src
|
|
p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools',
|
|
'aapt'),
|
|
'dump', 'badging', src],
|
|
stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
if thisbuild.get('novcheck', 'no') == "yes":
|
|
vercode = thisbuild['vercode']
|
|
version = thisbuild['version']
|
|
else:
|
|
vercode = None
|
|
version = None
|
|
for line in output.splitlines():
|
|
if line.startswith("package:"):
|
|
pat = re.compile(".*versionCode='([0-9]*)'.*")
|
|
vercode = re.match(pat, line).group(1)
|
|
pat = re.compile(".*versionName='([^']*)'.*")
|
|
version = re.match(pat, line).group(1)
|
|
if version == None or vercode == None:
|
|
print "Could not find version information in build in output"
|
|
sys.exit(1)
|
|
|
|
# Some apps (e.g. Timeriffic) have had the bonkers idea of
|
|
# including the entire changelog in the version number. Remove
|
|
# it so we can compare. (TODO: might be better to remove it
|
|
# before we compile, in fact)
|
|
index = version.find(" //")
|
|
if index != -1:
|
|
version = version[:index]
|
|
|
|
if (version != thisbuild['version'] or
|
|
vercode != thisbuild['vercode']):
|
|
print "Unexpected version/version code in output"
|
|
sys.exit(1)
|
|
|
|
# Copy the unsigned apk to our 'built' directory for further
|
|
# processing...
|
|
shutil.copyfile(src, dest_unsigned)
|
|
|
|
# Figure out the key alias name we'll use. Only the first 8
|
|
# characters are significant, so we'll use the first 8 from
|
|
# the MD5 of the app's ID and hope there are no collisions.
|
|
# If a collision does occur later, we're going to have to
|
|
# come up with a new alogrithm, AND rename all existing keys
|
|
# in the keystore!
|
|
if keyaliases.has_key(app['id']):
|
|
# For this particular app, the key alias is overridden...
|
|
keyalias = keyaliases[app['id']]
|
|
else:
|
|
m = md5.new()
|
|
m.update(app['id'])
|
|
keyalias = m.hexdigest()[:8]
|
|
print "Key alias: " + keyalias
|
|
|
|
# See if we already have a key for this application, and
|
|
# if not generate one...
|
|
p = subprocess.Popen(['keytool', '-list',
|
|
'-alias', keyalias, '-keystore', keystore,
|
|
'-storepass', keystorepass], stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
if p.returncode !=0:
|
|
print "Key does not exist - generating..."
|
|
p = subprocess.Popen(['keytool', '-genkey',
|
|
'-keystore', keystore, '-alias', keyalias,
|
|
'-keyalg', 'RSA', '-keysize', '2048',
|
|
'-validity', '10000',
|
|
'-storepass', keystorepass, '-keypass', keypass,
|
|
'-dname', keydname], stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
print output
|
|
if p.returncode != 0:
|
|
print "Failed to generate key"
|
|
sys.exit(1)
|
|
|
|
# Sign the application...
|
|
p = subprocess.Popen(['jarsigner', '-keystore', keystore,
|
|
'-storepass', keystorepass, '-keypass', keypass,
|
|
dest_unsigned, keyalias], stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
print output
|
|
if p.returncode != 0:
|
|
print "Failed to sign application"
|
|
sys.exit(1)
|
|
|
|
# Zipalign it...
|
|
p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'),
|
|
'-v', '4', dest_unsigned, dest],
|
|
stdout=subprocess.PIPE)
|
|
output = p.communicate()[0]
|
|
print output
|
|
if p.returncode != 0:
|
|
print "Failed to align application"
|
|
sys.exit(1)
|
|
os.remove(dest_unsigned)
|
|
|
|
print "Finished."
|
|
|