1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-17 10:40:12 +02:00

Got rid of source preparation code duplication

This commit is contained in:
Ciaran Gultnieks 2012-01-03 21:39:30 +00:00
parent bf3a9393cd
commit 262be1880d
3 changed files with 277 additions and 291 deletions

228
build.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -102,227 +102,11 @@ for app in apps:
else:
print "..building version " + thisbuild['version']
if not refreshed_source:
vcs.refreshlocal()
refreshed_source = True
# Optionally, the actual app source can be in a subdirectory...
if thisbuild.has_key('subdir'):
root_dir = os.path.join(build_dir, thisbuild['subdir'])
else:
root_dir = build_dir
# Get a working copy of the right revision...
if options.verbose:
print "Resetting repository to " + thisbuild['commit']
vcs.reset(thisbuild['commit'])
# Initialise submodules if requred...
if thisbuild.get('submodules', 'no') == 'yes':
vcs.initsubmodules()
# Generate (or update) the ant build file, build.xml...
if (thisbuild.get('update', 'yes') == 'yes' and
not thisbuild.has_key('maven')):
parms = [os.path.join(sdk_path, 'tools', 'android'),
'update', 'project', '-p', '.']
parms.append('--subprojects')
if thisbuild.has_key('target'):
parms.append('-t')
parms.append(thisbuild['target'])
if subprocess.call(parms, cwd=root_dir) != 0:
raise BuildException("Failed to update project")
# If the app has ant set up to sign the release, we need to switch
# that off, because we want the unsigned apk...
for propfile in ('build.properties', 'default.properties'):
if os.path.exists(os.path.join(root_dir, propfile)):
if subprocess.call(['sed','-i','s/^key.store/#/',
propfile], cwd=root_dir) !=0:
raise BuildException("Failed to amend %s" % propfile)
# Update the local.properties file...
locprops = os.path.join(root_dir, 'local.properties')
if os.path.exists(locprops):
f = open(locprops, 'r')
props = f.read()
f.close()
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary...
if thisbuild.get('oldsdkloc', 'no') == "yes":
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S|re.M).group(1)
props += "\nsdk-location=" + sdkloc + "\n"
# Add ndk location...
props+= "\nndk.dir=" + ndk_path + "\n"
# Add java.encoding if necessary...
if thisbuild.has_key('encoding'):
props += "\njava.encoding=" + thisbuild['encoding'] + "\n"
f = open(locprops, 'w')
f.write(props)
f.close()
# Insert version code and number into the manifest if necessary...
if thisbuild.has_key('insertversion'):
if subprocess.call(['sed','-i','s/' + thisbuild['insertversion'] +
'/' + thisbuild['version'] +'/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
if thisbuild.has_key('insertvercode'):
if subprocess.call(['sed','-i','s/' + thisbuild['insertvercode'] +
'/' + thisbuild['vercode'] +'/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
# Delete unwanted file...
if thisbuild.has_key('rm'):
os.remove(os.path.join(build_dir, thisbuild['rm']))
# Fix apostrophes translation files if necessary...
if thisbuild.get('fixapos', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
for filename in files:
if filename.endswith('.xml'):
if subprocess.call(['sed','-i','s@' +
r"\([^\\]\)'@\1\\'" +
'@g',
os.path.join(root, filename)]) != 0:
raise BuildException("Failed to amend " + filename)
# Fix translation files if necessary...
if thisbuild.get('fixtrans', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
for filename in files:
if filename.endswith('.xml'):
f = open(os.path.join(root, filename))
changed = False
outlines = []
for line in f:
num = 1
index = 0
oldline = line
while True:
index = line.find("%", index)
if index == -1:
break
next = line[index+1:index+2]
if next == "s" or next == "d":
line = (line[:index+1] +
str(num) + "$" +
line[index+1:])
num += 1
index += 3
else:
index += 1
# We only want to insert the positional arguments
# when there is more than one argument...
if oldline != line:
if num > 2:
changed = True
else:
line = oldline
outlines.append(line)
f.close()
if changed:
f = open(os.path.join(root, filename), 'w')
f.writelines(outlines)
f.close()
# Run a pre-build command if one is required...
if thisbuild.has_key('prebuild'):
if subprocess.call(thisbuild['prebuild'],
cwd=root_dir, shell=True) != 0:
raise BuildException("Error running pre-build command")
# Apply patches if any
if 'patch' in thisbuild:
for patch in thisbuild['patch'].split(';'):
print "Applying " + patch
patch_path = os.path.join('metadata', app['id'], patch)
if subprocess.call(['patch', '-p1',
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
raise BuildException("Failed to apply patch %s" % patch_path)
# Special case init functions for funambol...
if thisbuild.get('initfun', 'no') == "yes":
if subprocess.call(['sed','-i','s@' +
'<taskdef resource="net/sf/antcontrib/antcontrib.properties" />' +
'@' +
'<taskdef resource="net/sf/antcontrib/antcontrib.properties">' +
'<classpath>' +
'<pathelement location="/usr/share/java/ant-contrib.jar"/>' +
'</classpath>' +
'</taskdef>' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
if subprocess.call(['sed','-i','s@' +
'\${user.home}/funambol/build/android/build.properties' +
'@' +
'build.properties' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
buildxml = os.path.join(root_dir, 'build.xml')
f = open(buildxml, 'r')
xml = f.read()
f.close()
xmlout = ""
mode = 0
for line in xml.splitlines():
if mode == 0:
if line.find("jarsigner") != -1:
mode = 1
else:
xmlout += line + "\n"
else:
if line.find("/exec") != -1:
mode += 1
if mode == 3:
mode =0
f = open(buildxml, 'w')
f.write(xmlout)
f.close()
if subprocess.call(['sed','-i','s@' +
'platforms/android-2.0' +
'@' +
'platforms/android-8' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
shutil.copyfile(
os.path.join(root_dir, "build.properties.example"),
os.path.join(root_dir, "build.properties"))
if subprocess.call(['sed','-i','s@' +
'javacchome=.*'+
'@' +
'javacchome=' + javacc_path +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
if subprocess.call(['sed','-i','s@' +
'sdk-folder=.*'+
'@' +
'sdk-folder=' + sdk_path +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
if subprocess.call(['sed','-i','s@' +
'android.sdk.version.*'+
'@' +
'android.sdk.version=2.0' +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
# Prepare the source code...
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, sdk_path, ndk_path,
not refreshed_source)
refreshed_source = True
# Build the source tarball right before we build the release...
tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'

235
common.py
View File

@ -355,3 +355,238 @@ class MetaDataException(Exception):
def __str__(self):
return repr(self.value)
# Prepare the source code for a particular build
# 'vcs' - the appropriate vcs object for the application
# 'app' - the application details from the metadata
# 'build' - the build details from the metadata
# 'build_dir' - the path to the build directory
# 'sdk_path' - the path to the Android SDK
# 'ndk_path' - the path to the Android NDK
# 'refresh' - True to refresh from the remote repo
# Returns the root directory, which may be the same as 'build_dir' or may
# be a subdirectory of it.
def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh):
if refresh:
vcs.refreshlocal()
# Optionally, the actual app source can be in a subdirectory...
if build.has_key('subdir'):
root_dir = os.path.join(build_dir, build['subdir'])
else:
root_dir = build_dir
# Get a working copy of the right revision...
if options.verbose:
print "Resetting repository to " + build['commit']
vcs.reset(build['commit'])
# Initialise submodules if requred...
if build.get('submodules', 'no') == 'yes':
vcs.initsubmodules()
# Generate (or update) the ant build file, build.xml...
if (build.get('update', 'yes') == 'yes' and
not build.has_key('maven')):
parms = [os.path.join(sdk_path, 'tools', 'android'),
'update', 'project', '-p', '.']
parms.append('--subprojects')
if build.has_key('target'):
parms.append('-t')
parms.append(build['target'])
if subprocess.call(parms, cwd=root_dir) != 0:
raise BuildException("Failed to update project")
# If the app has ant set up to sign the release, we need to switch
# that off, because we want the unsigned apk...
for propfile in ('build.properties', 'default.properties'):
if os.path.exists(os.path.join(root_dir, propfile)):
if subprocess.call(['sed','-i','s/^key.store/#/',
propfile], cwd=root_dir) !=0:
raise BuildException("Failed to amend %s" % propfile)
# Update the local.properties file...
locprops = os.path.join(root_dir, 'local.properties')
if os.path.exists(locprops):
f = open(locprops, 'r')
props = f.read()
f.close()
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary...
if build.get('oldsdkloc', 'no') == "yes":
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S|re.M).group(1)
props += "\nsdk-location=" + sdkloc + "\n"
# Add ndk location...
props+= "\nndk.dir=" + ndk_path + "\n"
# Add java.encoding if necessary...
if build.has_key('encoding'):
props += "\njava.encoding=" + build['encoding'] + "\n"
f = open(locprops, 'w')
f.write(props)
f.close()
# Insert version code and number into the manifest if necessary...
if build.has_key('insertversion'):
if subprocess.call(['sed','-i','s/' + build['insertversion'] +
'/' + build['version'] +'/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
if build.has_key('insertvercode'):
if subprocess.call(['sed','-i','s/' + build['insertvercode'] +
'/' + build['vercode'] +'/g',
'AndroidManifest.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend manifest")
# Delete unwanted file...
if build.has_key('rm'):
os.remove(os.path.join(build_dir, build['rm']))
# Fix apostrophes translation files if necessary...
if build.get('fixapos', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
for filename in files:
if filename.endswith('.xml'):
if subprocess.call(['sed','-i','s@' +
r"\([^\\]\)'@\1\\'" +
'@g',
os.path.join(root, filename)]) != 0:
raise BuildException("Failed to amend " + filename)
# Fix translation files if necessary...
if build.get('fixtrans', 'no') == 'yes':
for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
for filename in files:
if filename.endswith('.xml'):
f = open(os.path.join(root, filename))
changed = False
outlines = []
for line in f:
num = 1
index = 0
oldline = line
while True:
index = line.find("%", index)
if index == -1:
break
next = line[index+1:index+2]
if next == "s" or next == "d":
line = (line[:index+1] +
str(num) + "$" +
line[index+1:])
num += 1
index += 3
else:
index += 1
# We only want to insert the positional arguments
# when there is more than one argument...
if oldline != line:
if num > 2:
changed = True
else:
line = oldline
outlines.append(line)
f.close()
if changed:
f = open(os.path.join(root, filename), 'w')
f.writelines(outlines)
f.close()
# Run a pre-build command if one is required...
if build.has_key('prebuild'):
if subprocess.call(build['prebuild'],
cwd=root_dir, shell=True) != 0:
raise BuildException("Error running pre-build command")
# Apply patches if any
if 'patch' in build:
for patch in build['patch'].split(';'):
print "Applying " + patch
patch_path = os.path.join('metadata', app['id'], patch)
if subprocess.call(['patch', '-p1',
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
raise BuildException("Failed to apply patch %s" % patch_path)
# Special case init functions for funambol...
if build.get('initfun', 'no') == "yes":
if subprocess.call(['sed','-i','s@' +
'<taskdef resource="net/sf/antcontrib/antcontrib.properties" />' +
'@' +
'<taskdef resource="net/sf/antcontrib/antcontrib.properties">' +
'<classpath>' +
'<pathelement location="/usr/share/java/ant-contrib.jar"/>' +
'</classpath>' +
'</taskdef>' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
if subprocess.call(['sed','-i','s@' +
'\${user.home}/funambol/build/android/build.properties' +
'@' +
'build.properties' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
buildxml = os.path.join(root_dir, 'build.xml')
f = open(buildxml, 'r')
xml = f.read()
f.close()
xmlout = ""
mode = 0
for line in xml.splitlines():
if mode == 0:
if line.find("jarsigner") != -1:
mode = 1
else:
xmlout += line + "\n"
else:
if line.find("/exec") != -1:
mode += 1
if mode == 3:
mode =0
f = open(buildxml, 'w')
f.write(xmlout)
f.close()
if subprocess.call(['sed','-i','s@' +
'platforms/android-2.0' +
'@' +
'platforms/android-8' +
'@g',
'build.xml'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.xml")
shutil.copyfile(
os.path.join(root_dir, "build.properties.example"),
os.path.join(root_dir, "build.properties"))
if subprocess.call(['sed','-i','s@' +
'javacchome=.*'+
'@' +
'javacchome=' + javacc_path +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
if subprocess.call(['sed','-i','s@' +
'sdk-folder=.*'+
'@' +
'sdk-folder=' + sdk_path +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
if subprocess.call(['sed','-i','s@' +
'android.sdk.version.*'+
'@' +
'android.sdk.version=2.0' +
'@g',
'build.properties'], cwd=root_dir) !=0:
raise BuildException("Failed to amend build.properties")
return root_dir

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# scanner.py - part of the FDroid server tools
@ -27,6 +27,8 @@ import subprocess
from optparse import OptionParser
import HTMLParser
import common
from common import BuildException
from common import VCSException
#Read configuration...
execfile('config.py')
@ -57,83 +59,48 @@ for app in apps:
print "Processing " + app['id']
build_dir = 'build/' + app['id']
try:
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
build_dir = 'build/' + app['id']
refreshed_source = False
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
refreshed_source = False
for thisbuild in app['builds']:
for thisbuild in app['builds']:
if thisbuild['commit'].startswith('!'):
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild['commit'][1:])
else:
print "..scanning version " + thisbuild['version']
if thisbuild['commit'].startswith('!'):
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild['commit'][1:])
else:
print "..scanning version " + thisbuild['version']
if not refreshed_source:
vcs.refreshlocal()
# Prepare the source code...
root_dir = common.prepare_source(vcs, app, thisbuild,
build_dir, sdk_path, ndk_path,
not refreshed_source)
refreshed_source = True
# Optionally, the actual app source can be in a subdirectory...
if thisbuild.has_key('subdir'):
root_dir = os.path.join(build_dir, thisbuild['subdir'])
else:
root_dir = build_dir
# Scan for common known non-free blobs:
usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar']
for r,d,f in os.walk(build_dir):
for curfile in f:
if curfile.lower() in usual_suspects:
msg = 'Found probable non-free blob ' + os.path.join(r,file)
msg += ' in ' + app['id'] + ' ' + thisbuild['version']
problems.append(msg)
# Get a working copy of the right revision...
if options.verbose:
print "Resetting repository to " + thisbuild['commit']
vcs.reset(thisbuild['commit'])
# Initialise submodules if requred...
if thisbuild.get('submodules', 'no') == 'yes':
vcs.initsubmodules()
# Generate (or update) the ant build file, build.xml...
if (thisbuild.get('update', 'yes') == 'yes' and
not thisbuild.has_key('maven')):
parms = [os.path.join(sdk_path, 'tools', 'android'),
'update', 'project', '-p', '.']
parms.append('--subprojects')
if thisbuild.has_key('target'):
parms.append('-t')
parms.append(thisbuild['target'])
if subprocess.call(parms, cwd=root_dir) != 0:
print "Failed to update project"
sys.exit(1)
# Delete unwanted file...
if thisbuild.has_key('rm'):
os.remove(os.path.join(build_dir, thisbuild['rm']))
# Run a pre-build command if one is required...
if thisbuild.has_key('prebuild'):
if subprocess.call(thisbuild['prebuild'],
cwd=root_dir, shell=True) != 0:
print "Error running pre-build command"
sys.exit(1)
# Apply patches if any
if 'patch' in thisbuild:
for patch in thisbuild['patch'].split(';'):
print "Applying " + patch
patch_path = os.path.join('metadata', app['id'], patch)
if subprocess.call(['patch', '-p1',
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
print "Failed to apply patch %s" % patch_path
sys.exit(1)
# Scan for common known non-free blobs:
usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar']
for r,d,f in os.walk(build_dir):
for curfile in f:
if curfile.lower() in usual_suspects:
msg = 'Found probable non-free blob ' + os.path.join(r,file)
msg += ' in ' + app['id'] + ' ' + thisbuild['version']
problems.append(msg)
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
problems.append(msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
problems.append(msg)
except Exception as e:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], e)
problems.append(msg)
print "Finished:"
for problem in problems: