From 262be1880dd7068e933771d350b0bc360e256866 Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Tue, 3 Jan 2012 21:39:30 +0000 Subject: [PATCH] Got rid of source preparation code duplication --- build.py | 228 ++------------------------------------------------- common.py | 235 +++++++++++++++++++++++++++++++++++++++++++++++++++++ scanner.py | 105 ++++++++---------------- 3 files changed, 277 insertions(+), 291 deletions(-) diff --git a/build.py b/build.py index a7b09139..d4ba7004 100755 --- a/build.py +++ b/build.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # build.py - part of the FDroid server tools -# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -102,227 +102,11 @@ for app in apps: else: print "..building version " + thisbuild['version'] - if not refreshed_source: - vcs.refreshlocal() - refreshed_source = True - - # Optionally, the actual app source can be in a subdirectory... - if thisbuild.has_key('subdir'): - root_dir = os.path.join(build_dir, thisbuild['subdir']) - else: - root_dir = build_dir - - # Get a working copy of the right revision... - if options.verbose: - print "Resetting repository to " + thisbuild['commit'] - vcs.reset(thisbuild['commit']) - - # Initialise submodules if requred... - if thisbuild.get('submodules', 'no') == 'yes': - vcs.initsubmodules() - - # Generate (or update) the ant build file, build.xml... - if (thisbuild.get('update', 'yes') == 'yes' and - not thisbuild.has_key('maven')): - parms = [os.path.join(sdk_path, 'tools', 'android'), - 'update', 'project', '-p', '.'] - parms.append('--subprojects') - if thisbuild.has_key('target'): - parms.append('-t') - parms.append(thisbuild['target']) - if subprocess.call(parms, cwd=root_dir) != 0: - raise BuildException("Failed to update project") - - # If the app has ant set up to sign the release, we need to switch - # that off, because we want the unsigned apk... - for propfile in ('build.properties', 'default.properties'): - if os.path.exists(os.path.join(root_dir, propfile)): - if subprocess.call(['sed','-i','s/^key.store/#/', - propfile], cwd=root_dir) !=0: - raise BuildException("Failed to amend %s" % propfile) - - # Update the local.properties file... - locprops = os.path.join(root_dir, 'local.properties') - if os.path.exists(locprops): - f = open(locprops, 'r') - props = f.read() - f.close() - # Fix old-fashioned 'sdk-location' by copying - # from sdk.dir, if necessary... - if thisbuild.get('oldsdkloc', 'no') == "yes": - sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props, - re.S|re.M).group(1) - props += "\nsdk-location=" + sdkloc + "\n" - # Add ndk location... - props+= "\nndk.dir=" + ndk_path + "\n" - # Add java.encoding if necessary... - if thisbuild.has_key('encoding'): - props += "\njava.encoding=" + thisbuild['encoding'] + "\n" - f = open(locprops, 'w') - f.write(props) - f.close() - - # Insert version code and number into the manifest if necessary... - if thisbuild.has_key('insertversion'): - if subprocess.call(['sed','-i','s/' + thisbuild['insertversion'] + - '/' + thisbuild['version'] +'/g', - 'AndroidManifest.xml'], cwd=root_dir) !=0: - raise BuildException("Failed to amend manifest") - if thisbuild.has_key('insertvercode'): - if subprocess.call(['sed','-i','s/' + thisbuild['insertvercode'] + - '/' + thisbuild['vercode'] +'/g', - 'AndroidManifest.xml'], cwd=root_dir) !=0: - raise BuildException("Failed to amend manifest") - - # Delete unwanted file... - if thisbuild.has_key('rm'): - os.remove(os.path.join(build_dir, thisbuild['rm'])) - - # Fix apostrophes translation files if necessary... - if thisbuild.get('fixapos', 'no') == 'yes': - for root, dirs, files in os.walk(os.path.join(root_dir,'res')): - for filename in files: - if filename.endswith('.xml'): - if subprocess.call(['sed','-i','s@' + - r"\([^\\]\)'@\1\\'" + - '@g', - os.path.join(root, filename)]) != 0: - raise BuildException("Failed to amend " + filename) - - # Fix translation files if necessary... - if thisbuild.get('fixtrans', 'no') == 'yes': - for root, dirs, files in os.walk(os.path.join(root_dir,'res')): - for filename in files: - if filename.endswith('.xml'): - f = open(os.path.join(root, filename)) - changed = False - outlines = [] - for line in f: - num = 1 - index = 0 - oldline = line - while True: - index = line.find("%", index) - if index == -1: - break - next = line[index+1:index+2] - if next == "s" or next == "d": - line = (line[:index+1] + - str(num) + "$" + - line[index+1:]) - num += 1 - index += 3 - else: - index += 1 - # We only want to insert the positional arguments - # when there is more than one argument... - if oldline != line: - if num > 2: - changed = True - else: - line = oldline - outlines.append(line) - f.close() - if changed: - f = open(os.path.join(root, filename), 'w') - f.writelines(outlines) - f.close() - - # Run a pre-build command if one is required... - if thisbuild.has_key('prebuild'): - if subprocess.call(thisbuild['prebuild'], - cwd=root_dir, shell=True) != 0: - raise BuildException("Error running pre-build command") - - # Apply patches if any - if 'patch' in thisbuild: - for patch in thisbuild['patch'].split(';'): - print "Applying " + patch - patch_path = os.path.join('metadata', app['id'], patch) - if subprocess.call(['patch', '-p1', - '-i', os.path.abspath(patch_path)], cwd=build_dir) != 0: - raise BuildException("Failed to apply patch %s" % patch_path) - - # Special case init functions for funambol... - if thisbuild.get('initfun', 'no') == "yes": - - if subprocess.call(['sed','-i','s@' + - '' + - '@' + - '' + - '' + - '' + - '' + - '' + - '@g', - 'build.xml'], cwd=root_dir) !=0: - raise BuildException("Failed to amend build.xml") - - if subprocess.call(['sed','-i','s@' + - '\${user.home}/funambol/build/android/build.properties' + - '@' + - 'build.properties' + - '@g', - 'build.xml'], cwd=root_dir) !=0: - raise BuildException("Failed to amend build.xml") - - buildxml = os.path.join(root_dir, 'build.xml') - f = open(buildxml, 'r') - xml = f.read() - f.close() - xmlout = "" - mode = 0 - for line in xml.splitlines(): - if mode == 0: - if line.find("jarsigner") != -1: - mode = 1 - else: - xmlout += line + "\n" - else: - if line.find("/exec") != -1: - mode += 1 - if mode == 3: - mode =0 - f = open(buildxml, 'w') - f.write(xmlout) - f.close() - - if subprocess.call(['sed','-i','s@' + - 'platforms/android-2.0' + - '@' + - 'platforms/android-8' + - '@g', - 'build.xml'], cwd=root_dir) !=0: - raise BuildException("Failed to amend build.xml") - - shutil.copyfile( - os.path.join(root_dir, "build.properties.example"), - os.path.join(root_dir, "build.properties")) - - if subprocess.call(['sed','-i','s@' + - 'javacchome=.*'+ - '@' + - 'javacchome=' + javacc_path + - '@g', - 'build.properties'], cwd=root_dir) !=0: - raise BuildException("Failed to amend build.properties") - - if subprocess.call(['sed','-i','s@' + - 'sdk-folder=.*'+ - '@' + - 'sdk-folder=' + sdk_path + - '@g', - 'build.properties'], cwd=root_dir) !=0: - raise BuildException("Failed to amend build.properties") - - if subprocess.call(['sed','-i','s@' + - 'android.sdk.version.*'+ - '@' + - 'android.sdk.version=2.0' + - '@g', - 'build.properties'], cwd=root_dir) !=0: - raise BuildException("Failed to amend build.properties") - + # Prepare the source code... + root_dir = common.prepare_source(vcs, app, thisbuild, + build_dir, sdk_path, ndk_path, + not refreshed_source) + refreshed_source = True # Build the source tarball right before we build the release... tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' diff --git a/common.py b/common.py index a7d4d613..a1abfeca 100644 --- a/common.py +++ b/common.py @@ -355,3 +355,238 @@ class MetaDataException(Exception): def __str__(self): return repr(self.value) + +# Prepare the source code for a particular build +# 'vcs' - the appropriate vcs object for the application +# 'app' - the application details from the metadata +# 'build' - the build details from the metadata +# 'build_dir' - the path to the build directory +# 'sdk_path' - the path to the Android SDK +# 'ndk_path' - the path to the Android NDK +# 'refresh' - True to refresh from the remote repo +# Returns the root directory, which may be the same as 'build_dir' or may +# be a subdirectory of it. +def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh): + + if refresh: + vcs.refreshlocal() + + # Optionally, the actual app source can be in a subdirectory... + if build.has_key('subdir'): + root_dir = os.path.join(build_dir, build['subdir']) + else: + root_dir = build_dir + + # Get a working copy of the right revision... + if options.verbose: + print "Resetting repository to " + build['commit'] + vcs.reset(build['commit']) + + # Initialise submodules if requred... + if build.get('submodules', 'no') == 'yes': + vcs.initsubmodules() + + # Generate (or update) the ant build file, build.xml... + if (build.get('update', 'yes') == 'yes' and + not build.has_key('maven')): + parms = [os.path.join(sdk_path, 'tools', 'android'), + 'update', 'project', '-p', '.'] + parms.append('--subprojects') + if build.has_key('target'): + parms.append('-t') + parms.append(build['target']) + if subprocess.call(parms, cwd=root_dir) != 0: + raise BuildException("Failed to update project") + + # If the app has ant set up to sign the release, we need to switch + # that off, because we want the unsigned apk... + for propfile in ('build.properties', 'default.properties'): + if os.path.exists(os.path.join(root_dir, propfile)): + if subprocess.call(['sed','-i','s/^key.store/#/', + propfile], cwd=root_dir) !=0: + raise BuildException("Failed to amend %s" % propfile) + + # Update the local.properties file... + locprops = os.path.join(root_dir, 'local.properties') + if os.path.exists(locprops): + f = open(locprops, 'r') + props = f.read() + f.close() + # Fix old-fashioned 'sdk-location' by copying + # from sdk.dir, if necessary... + if build.get('oldsdkloc', 'no') == "yes": + sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props, + re.S|re.M).group(1) + props += "\nsdk-location=" + sdkloc + "\n" + # Add ndk location... + props+= "\nndk.dir=" + ndk_path + "\n" + # Add java.encoding if necessary... + if build.has_key('encoding'): + props += "\njava.encoding=" + build['encoding'] + "\n" + f = open(locprops, 'w') + f.write(props) + f.close() + + # Insert version code and number into the manifest if necessary... + if build.has_key('insertversion'): + if subprocess.call(['sed','-i','s/' + build['insertversion'] + + '/' + build['version'] +'/g', + 'AndroidManifest.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend manifest") + if build.has_key('insertvercode'): + if subprocess.call(['sed','-i','s/' + build['insertvercode'] + + '/' + build['vercode'] +'/g', + 'AndroidManifest.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend manifest") + + # Delete unwanted file... + if build.has_key('rm'): + os.remove(os.path.join(build_dir, build['rm'])) + + # Fix apostrophes translation files if necessary... + if build.get('fixapos', 'no') == 'yes': + for root, dirs, files in os.walk(os.path.join(root_dir,'res')): + for filename in files: + if filename.endswith('.xml'): + if subprocess.call(['sed','-i','s@' + + r"\([^\\]\)'@\1\\'" + + '@g', + os.path.join(root, filename)]) != 0: + raise BuildException("Failed to amend " + filename) + + # Fix translation files if necessary... + if build.get('fixtrans', 'no') == 'yes': + for root, dirs, files in os.walk(os.path.join(root_dir,'res')): + for filename in files: + if filename.endswith('.xml'): + f = open(os.path.join(root, filename)) + changed = False + outlines = [] + for line in f: + num = 1 + index = 0 + oldline = line + while True: + index = line.find("%", index) + if index == -1: + break + next = line[index+1:index+2] + if next == "s" or next == "d": + line = (line[:index+1] + + str(num) + "$" + + line[index+1:]) + num += 1 + index += 3 + else: + index += 1 + # We only want to insert the positional arguments + # when there is more than one argument... + if oldline != line: + if num > 2: + changed = True + else: + line = oldline + outlines.append(line) + f.close() + if changed: + f = open(os.path.join(root, filename), 'w') + f.writelines(outlines) + f.close() + + # Run a pre-build command if one is required... + if build.has_key('prebuild'): + if subprocess.call(build['prebuild'], + cwd=root_dir, shell=True) != 0: + raise BuildException("Error running pre-build command") + + # Apply patches if any + if 'patch' in build: + for patch in build['patch'].split(';'): + print "Applying " + patch + patch_path = os.path.join('metadata', app['id'], patch) + if subprocess.call(['patch', '-p1', + '-i', os.path.abspath(patch_path)], cwd=build_dir) != 0: + raise BuildException("Failed to apply patch %s" % patch_path) + + # Special case init functions for funambol... + if build.get('initfun', 'no') == "yes": + + if subprocess.call(['sed','-i','s@' + + '' + + '@' + + '' + + '' + + '' + + '' + + '' + + '@g', + 'build.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.xml") + + if subprocess.call(['sed','-i','s@' + + '\${user.home}/funambol/build/android/build.properties' + + '@' + + 'build.properties' + + '@g', + 'build.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.xml") + + buildxml = os.path.join(root_dir, 'build.xml') + f = open(buildxml, 'r') + xml = f.read() + f.close() + xmlout = "" + mode = 0 + for line in xml.splitlines(): + if mode == 0: + if line.find("jarsigner") != -1: + mode = 1 + else: + xmlout += line + "\n" + else: + if line.find("/exec") != -1: + mode += 1 + if mode == 3: + mode =0 + f = open(buildxml, 'w') + f.write(xmlout) + f.close() + + if subprocess.call(['sed','-i','s@' + + 'platforms/android-2.0' + + '@' + + 'platforms/android-8' + + '@g', + 'build.xml'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.xml") + + shutil.copyfile( + os.path.join(root_dir, "build.properties.example"), + os.path.join(root_dir, "build.properties")) + + if subprocess.call(['sed','-i','s@' + + 'javacchome=.*'+ + '@' + + 'javacchome=' + javacc_path + + '@g', + 'build.properties'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.properties") + + if subprocess.call(['sed','-i','s@' + + 'sdk-folder=.*'+ + '@' + + 'sdk-folder=' + sdk_path + + '@g', + 'build.properties'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.properties") + + if subprocess.call(['sed','-i','s@' + + 'android.sdk.version.*'+ + '@' + + 'android.sdk.version=2.0' + + '@g', + 'build.properties'], cwd=root_dir) !=0: + raise BuildException("Failed to amend build.properties") + + return root_dir + diff --git a/scanner.py b/scanner.py index 843c287f..32fd0a38 100755 --- a/scanner.py +++ b/scanner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- # # scanner.py - part of the FDroid server tools @@ -27,6 +27,8 @@ import subprocess from optparse import OptionParser import HTMLParser import common +from common import BuildException +from common import VCSException #Read configuration... execfile('config.py') @@ -57,83 +59,48 @@ for app in apps: print "Processing " + app['id'] - build_dir = 'build/' + app['id'] + try: - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['repotype'], app['repo'], build_dir) + build_dir = 'build/' + app['id'] - refreshed_source = False + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app['repotype'], app['repo'], build_dir) + + refreshed_source = False - for thisbuild in app['builds']: + for thisbuild in app['builds']: - if thisbuild['commit'].startswith('!'): - print ("..skipping version " + thisbuild['version'] + " - " + - thisbuild['commit'][1:]) - else: - print "..scanning version " + thisbuild['version'] + if thisbuild['commit'].startswith('!'): + print ("..skipping version " + thisbuild['version'] + " - " + + thisbuild['commit'][1:]) + else: + print "..scanning version " + thisbuild['version'] - if not refreshed_source: - vcs.refreshlocal() + # Prepare the source code... + root_dir = common.prepare_source(vcs, app, thisbuild, + build_dir, sdk_path, ndk_path, + not refreshed_source) refreshed_source = True - # Optionally, the actual app source can be in a subdirectory... - if thisbuild.has_key('subdir'): - root_dir = os.path.join(build_dir, thisbuild['subdir']) - else: - root_dir = build_dir + # Scan for common known non-free blobs: + usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar'] + for r,d,f in os.walk(build_dir): + for curfile in f: + if curfile.lower() in usual_suspects: + msg = 'Found probable non-free blob ' + os.path.join(r,file) + msg += ' in ' + app['id'] + ' ' + thisbuild['version'] + problems.append(msg) - # Get a working copy of the right revision... - if options.verbose: - print "Resetting repository to " + thisbuild['commit'] - vcs.reset(thisbuild['commit']) - - # Initialise submodules if requred... - if thisbuild.get('submodules', 'no') == 'yes': - vcs.initsubmodules() - - # Generate (or update) the ant build file, build.xml... - if (thisbuild.get('update', 'yes') == 'yes' and - not thisbuild.has_key('maven')): - parms = [os.path.join(sdk_path, 'tools', 'android'), - 'update', 'project', '-p', '.'] - parms.append('--subprojects') - if thisbuild.has_key('target'): - parms.append('-t') - parms.append(thisbuild['target']) - if subprocess.call(parms, cwd=root_dir) != 0: - print "Failed to update project" - sys.exit(1) - - # Delete unwanted file... - if thisbuild.has_key('rm'): - os.remove(os.path.join(build_dir, thisbuild['rm'])) - - # Run a pre-build command if one is required... - if thisbuild.has_key('prebuild'): - if subprocess.call(thisbuild['prebuild'], - cwd=root_dir, shell=True) != 0: - print "Error running pre-build command" - sys.exit(1) - - # Apply patches if any - if 'patch' in thisbuild: - for patch in thisbuild['patch'].split(';'): - print "Applying " + patch - patch_path = os.path.join('metadata', app['id'], patch) - if subprocess.call(['patch', '-p1', - '-i', os.path.abspath(patch_path)], cwd=build_dir) != 0: - print "Failed to apply patch %s" % patch_path - sys.exit(1) - - # Scan for common known non-free blobs: - usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar'] - for r,d,f in os.walk(build_dir): - for curfile in f: - if curfile.lower() in usual_suspects: - msg = 'Found probable non-free blob ' + os.path.join(r,file) - msg += ' in ' + app['id'] + ' ' + thisbuild['version'] - problems.append(msg) + except BuildException as be: + msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be) + problems.append(msg) + except VCSException as vcse: + msg = "VCS error while scanning app %s: %s" % (app['id'], vcse) + problems.append(msg) + except Exception as e: + msg = "Could not scan app %s due to unknown error: %s" % (app['id'], e) + problems.append(msg) print "Finished:" for problem in problems: