# -*- coding: utf-8 -*-
#
# common.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see
'
self.state = self.stPARA
elif self.state == self.stPARA:
self.text_html += ' '
self.text_plain += ' '
self.addtext(line)
def end(self):
self.endcur()
# Parse multiple lines of description as written in a metadata file, returning
# a single string in plain text format.
def description_plain(lines, linkres):
ps = DescriptionFormatter(linkres)
for line in lines:
ps.parseline(line)
ps.end()
return ps.text_plain
# Parse multiple lines of description as written in a metadata file, returning
# a single string in wiki format.
def description_wiki(lines):
ps = DescriptionFormatter(None)
for line in lines:
ps.parseline(line)
ps.end()
return ps.text_wiki
# Parse multiple lines of description as written in a metadata file, returning
# a single string in HTML format.
def description_html(lines,linkres):
ps = DescriptionFormatter(linkres)
for line in lines:
ps.parseline(line)
ps.end()
return ps.text_html
def retrieve_string(xml_dir, string):
if not string.startswith('@string/'):
return string.replace("\\'","'")
string_search = re.compile(r'.*"'+string[8:]+'".*>([^<]+?)<.*').search
for xmlfile in glob.glob(os.path.join(xml_dir, '*.xml')):
for line in file(xmlfile):
matches = string_search(line)
if matches:
return retrieve_string(xml_dir, matches.group(1))
return ''
# Return list of existing files that will be used to find the highest vercode
def manifest_paths(app_dir, flavour):
possible_manifests = [ os.path.join(app_dir, 'AndroidManifest.xml'),
os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
os.path.join(app_dir, 'build.gradle') ]
if flavour is not None:
possible_manifests.append(
os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
return [path for path in possible_manifests if os.path.isfile(path)]
# Retrieve the package name
def fetch_real_name(app_dir, flavour):
app_search = re.compile(r'.*\n"
ret += str(self.stdout)
ret += "
\n"
if self.stderr:
ret += "=stderr=\n"
ret += "\n"
ret += str(self.stderr)
ret += "
\n"
return ret
def __str__(self):
ret = repr(self.value)
if self.stdout:
ret = ret + "\n==== stdout begin ====\n" + str(self.stdout) + "\n==== stdout end ===="
if self.stderr:
ret = ret + "\n==== stderr begin ====\n" + str(self.stderr) + "\n==== stderr end ===="
return ret
class VCSException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class MetaDataException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def parse_srclib(metafile, **kw):
thisinfo = {}
if metafile and not isinstance(metafile, file):
metafile = open(metafile, "r")
# Defaults for fields that come from metadata
thisinfo['Repo Type'] = ''
thisinfo['Repo'] = ''
thisinfo['Subdir'] = None
thisinfo['Prepare'] = None
thisinfo['Update Project'] = None
if metafile is None:
return thisinfo
mode = 0
buildlines = []
for line in metafile:
line = line.rstrip('\r\n')
if len(line) == 0:
continue
if line.startswith("#"):
continue
index = line.find(':')
if index == -1:
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
field = line[:index]
value = line[index+1:]
if field == "Subdir":
thisinfo[field] = value.split(',')
else:
thisinfo[field] = value
return thisinfo
# Get the specified source library.
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
def getsrclib(spec, srclib_dir, sdk_path, ndk_path="", mvn3="", basepath=False, raw=False, prepare=True, preponly=False):
if raw:
name = spec
ref = None
else:
name, ref = spec.split('@')
srclib_path = os.path.join('srclibs', name + ".txt")
if not os.path.exists(srclib_path):
raise BuildException('srclib ' + name + ' not found.')
srclib = parse_srclib(srclib_path)
sdir = os.path.join(srclib_dir, name)
if not preponly:
vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir, sdk_path)
vcs.srclib = (name, sdir)
vcs.gotorevision(ref)
if raw:
return vcs
libdir = None
if srclib["Subdir"] is not None:
for subdir in srclib["Subdir"]:
libdir_candidate = os.path.join(sdir, subdir)
if os.path.exists(libdir_candidate):
libdir = libdir_candidate
break
if libdir is None:
libdir = sdir
if prepare:
if srclib["Prepare"] is not None:
cmd = srclib["Prepare"].replace('$$SDK$$', sdk_path)
cmd = cmd.replace('$$NDK$$', ndk_path).replace('$$MVN$$', mvn3)
print "******************************* PREPARE " + cmd + " **************"
p = subprocess.Popen(['bash', '-c', cmd], cwd=libdir,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if p.returncode != 0:
raise BuildException("Error running prepare command for srclib "
+ name, out, err)
if srclib["Update Project"] == "Yes":
print "Updating srclib %s at path %s" % (name, libdir)
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
'update', 'project', '-p', libdir]) != 0:
raise BuildException( 'Error updating ' + name + ' project')
if basepath:
return sdir
return libdir
# Prepare the source code for a particular build
# 'vcs' - the appropriate vcs object for the application
# 'app' - the application details from the metadata
# 'build' - the build details from the metadata
# 'build_dir' - the path to the build directory, usually
# 'build/app.id'
# 'srclib_dir' - the path to the source libraries directory, usually
# 'build/srclib'
# 'extlib_dir' - the path to the external libraries directory, usually
# 'build/extlib'
# 'sdk_path' - the path to the Android SDK
# 'ndk_path' - the path to the Android NDK
# 'javacc_path' - the path to javacc
# 'mvn3' - the path to the maven 3 executable
# 'verbose' - optional: verbose or not (default=False)
# Returns the (root, srclibpaths) where:
# 'root' is the root directory, which may be the same as 'build_dir' or may
# be a subdirectory of it.
# 'srclibpaths' is information on the srclibs being used
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, verbose=False, onserver=False):
# Optionally, the actual app source can be in a subdirectory...
if 'subdir' in build:
root_dir = os.path.join(build_dir, build['subdir'])
else:
root_dir = build_dir
# Get a working copy of the right revision...
print "Getting source for revision " + build['commit']
vcs.gotorevision(build['commit'])
# Check that a subdir (if we're using one) exists. This has to happen
# after the checkout, since it might not exist elsewhere...
if not os.path.exists(root_dir):
raise BuildException('Missing subdir ' + root_dir)
# Initialise submodules if requred...
if build.get('submodules', 'no') == 'yes':
if verbose: print "Initialising submodules..."
vcs.initsubmodules()
# Run an init command if one is required...
if 'init' in build:
init = build['init']
init = init.replace('$$SDK$$', sdk_path)
init = init.replace('$$NDK$$', ndk_path)
init = init.replace('$$MVN$$', mvn3)
if verbose: print "Doing init: exec '%s' in '%s'"%(init,root_dir)
if subprocess.call(['bash', '-c', init], cwd=root_dir) != 0:
raise BuildException("Error running init command")
# Generate (or update) the ant build file, build.xml...
updatemode = build.get('update', '.')
if (updatemode != 'no' and
'maven' not in build and 'gradle' not in build):
parms = [os.path.join(sdk_path, 'tools', 'android'),
'update', 'project', '-p', '.']
parms.append('--subprojects')
if 'target' in build:
parms.append('-t')
parms.append(build['target'])
update_dirs = updatemode.split(';')
# Force build.xml update if necessary...
if updatemode == 'force' or 'target' in build:
if updatemode == 'force':
update_dirs = ['.']
buildxml = os.path.join(root_dir, 'build.xml')
if os.path.exists(buildxml):
print 'Force-removing old build.xml'
os.remove(buildxml)
for d in update_dirs:
cwd = os.path.join(root_dir, d)
if verbose:
print "Update of '%s': exec '%s' in '%s'"%\
(d," ".join(parms),cwd)
p = subprocess.Popen(parms, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = p.communicate()
if p.returncode != 0:
raise BuildException("Failed to update project with stdout '%s' and stderr '%s'"%(out,err))
# check to see whether an error was returned without a proper exit code (this is the case for the 'no target set or target invalid' error)
if err != "" and err.startswith("Error: "):
raise BuildException("Failed to update project with stdout '%s' and stderr '%s'"%(out,err))
# If the app has ant set up to sign the release, we need to switch
# that off, because we want the unsigned apk...
for propfile in ('build.properties', 'default.properties', 'ant.properties'):
if os.path.exists(os.path.join(root_dir, propfile)):
if subprocess.call(['sed','-i','s/^key.store/#/',
propfile], cwd=root_dir) !=0:
raise BuildException("Failed to amend %s" % propfile)
# Update the local.properties file...
locprops = os.path.join(root_dir, 'local.properties')
if os.path.exists(locprops):
f = open(locprops, 'r')
props = f.read()
f.close()
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary...
if build.get('oldsdkloc', 'no') == "yes":
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S|re.M).group(1)
props += "\nsdk-location=" + sdkloc + "\n"
# Add ndk location...
props+= "\nndk.dir=" + ndk_path + "\n"
# Add java.encoding if necessary...
if 'encoding' in build:
props += "\njava.encoding=" + build['encoding'] + "\n"
f = open(locprops, 'w')
f.write(props)
f.close()
# Insert version code and number into the manifest if necessary...
if 'forceversion' in build:
if subprocess.call(['sed','-r','-i',
's/android:versionName="[^"]+"/android:versionName="' + build['version'] + '"/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
if 'forcevercode' in build:
if subprocess.call(['sed','-r','-i',
's/android:versionCode="[^"]+"/android:versionCode="' + build['vercode'] + '"/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
# Delete unwanted file...
if 'rm' in build:
for part in build['rm'].split(';'):
dest = os.path.join(build_dir, part)
if os.path.exists(dest):
os.remove(dest)
# Fix apostrophes translation files if necessary...
if build.get('fixapos', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
for filename in files:
if filename.endswith('.xml'):
if subprocess.call(['sed','-i','s@' +
r"\([^\\]\)'@\1\\'" +
'@g',
os.path.join(root, filename)]) != 0:
raise BuildException("Failed to amend " + filename)
# Fix translation files if necessary...
if build.get('fixtrans', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
for filename in files:
if filename.endswith('.xml'):
f = open(os.path.join(root, filename))
changed = False
outlines = []
for line in f:
num = 1
index = 0
oldline = line
while True:
index = line.find("%", index)
if index == -1:
break
next = line[index+1:index+2]
if next == "s" or next == "d":
line = (line[:index+1] +
str(num) + "$" +
line[index+1:])
num += 1
index += 3
else:
index += 1
# We only want to insert the positional arguments
# when there is more than one argument...
if oldline != line:
if num > 2:
changed = True
else:
line = oldline
outlines.append(line)
f.close()
if changed:
f = open(os.path.join(root, filename), 'w')
f.writelines(outlines)
f.close()
# Add required external libraries...
if 'extlibs' in build:
libsdir = os.path.join(root_dir, 'libs')
if not os.path.exists(libsdir):
os.mkdir(libsdir)
for lib in build['extlibs'].split(';'):
libf = os.path.basename(lib)
shutil.copyfile(os.path.join(extlib_dir, lib),
os.path.join(libsdir, libf))
# Get required source libraries...
srclibpaths = []
if 'srclibs' in build:
for lib in build['srclibs'].split(';'):
name, _ = lib.split('@')
srclibpaths.append((name, getsrclib(lib, srclib_dir, sdk_path, ndk_path, mvn3, preponly=onserver)))
basesrclib = vcs.getsrclib()
# If one was used for the main source, add that too.
if basesrclib:
srclibpaths.append(basesrclib)
# There should never be bin, gen or native libs directories in the source, so just get
# rid of them...
for baddir in ['gen', 'bin', 'obj', 'libs/armeabi-v7a', 'libs/armeabi', 'libs/mips', 'libs/x86']:
badpath = os.path.join(root_dir, baddir)
if os.path.exists(badpath):
shutil.rmtree(badpath)
# Apply patches if any
if 'patch' in build:
for patch in build['patch'].split(';'):
print "Applying " + patch
patch_path = os.path.join('metadata', app['id'], patch)
if subprocess.call(['patch', '-p1',
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
raise BuildException("Failed to apply patch %s" % patch_path)
# Run a pre-build command if one is required...
if 'prebuild' in build:
prebuild = build['prebuild']
# Substitute source library paths into prebuild commands...
for name, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
prebuild = prebuild.replace('$$' + name + '$$', libpath)
prebuild = prebuild.replace('$$SDK$$', sdk_path)
prebuild = prebuild.replace('$$NDK$$', ndk_path)
prebuild = prebuild.replace('$$MVN3$$', mvn3)
p = subprocess.Popen(['bash', '-c', prebuild], cwd=root_dir,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if p.returncode != 0:
raise BuildException("Error running pre-build command", out, err)
if build.get('anal-tics', 'no') == 'yes':
fp = os.path.join(root_dir, 'src', 'com', 'google', 'android', 'apps', 'analytics')
os.makedirs(fp)
with open(os.path.join(fp, 'GoogleAnalyticsTracker.java'), 'w') as f:
f.write("""
package com.google.android.apps.analytics;
public class GoogleAnalyticsTracker {
private static GoogleAnalyticsTracker instance;
private GoogleAnalyticsTracker() {
}
public static GoogleAnalyticsTracker getInstance() {
if(instance == null)
instance = new GoogleAnalyticsTracker();
return instance;
}
public void start(String i,int think ,Object not) {
}
public void dispatch() {
}
public void stop() {
}
public void setProductVersion(String uh, String hu) {
}
public void trackEvent(String that,String just,String aint,int happening) {
}
public void trackPageView(String nope) {
}
public void setCustomVar(int mind,String your,String own,int business) {
}
}
""")
# Special case init functions for funambol...
if build.get('initfun', 'no') == "yes":
if subprocess.call(['sed','-i','s@' +
'