From 3d72c30fe59e7bc96a166258864ffa5b070755d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sat, 31 May 2014 23:10:16 +0200 Subject: [PATCH] Use ordered dicts for defaults in apps and builds --- fdroidserver/build.py | 34 ++++---- fdroidserver/checkupdates.py | 17 ++-- fdroidserver/common.py | 39 +++++---- fdroidserver/lint.py | 20 +++-- fdroidserver/metadata.py | 150 ++++++++++++++++++++--------------- fdroidserver/scanner.py | 2 +- fdroidserver/update.py | 4 +- 7 files changed, 141 insertions(+), 125 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 45eac1ec..f0a24b6b 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -326,7 +326,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force): ftp.mkdir('extlib') ftp.mkdir('srclib') # Copy any extlibs that are required... - if 'extlibs' in thisbuild: + if thisbuild['extlibs']: ftp.chdir(homedir + '/build/extlib') for lib in thisbuild['extlibs']: lib = lib.strip() @@ -343,7 +343,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force): ftp.chdir('..') # Copy any srclibs that are required... srclibpaths = [] - if 'srclibs' in thisbuild: + if thisbuild['srclibs']: for lib in thisbuild['srclibs']: srclibpaths.append( common.getsrclib(lib, 'build/srclib', srclibpaths, @@ -439,7 +439,7 @@ def adapt_gradle(build_dir): def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver): """Do a build locally.""" - if thisbuild.get('buildjni') not in (None, ['no']): + if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: if not config['ndk_path']: logging.critical("$ANDROID_NDK is not set!") sys.exit(3) @@ -546,13 +546,14 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d f.write(manifestcontent) # Run a build command if one is required... - if 'build' in thisbuild: + if thisbuild['build']: + logging.info("Running 'build' commands in %s" % root_dir) cmd = common.replace_config_vars(thisbuild['build']) + # Substitute source library paths into commands... for name, number, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) - logging.info("Running 'build' commands in %s" % root_dir) p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) @@ -561,9 +562,10 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d (app['id'], thisbuild['version']), p.stdout) # Build native stuff if required... - if thisbuild.get('buildjni') not in (None, ['no']): - logging.info("Building native libraries...") - jni_components = thisbuild.get('buildjni') + if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: + logging.info("Building the native code") + jni_components = thisbuild['buildjni'] + if jni_components == ['yes']: jni_components = [''] cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"] @@ -601,7 +603,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d '-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true', '-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package'] - if 'target' in thisbuild: + if thisbuild['target']: target = thisbuild["target"].split('-')[1] FDroidPopen(['sed', '-i', 's@[0-9]*@' @@ -615,7 +617,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d 'pom.xml'], cwd=maven_dir) - if 'mvnflags' in thisbuild: + if thisbuild['mvnflags']: mvncmd += thisbuild['mvnflags'] p = FDroidPopen(mvncmd, cwd=maven_dir) @@ -697,7 +699,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d flavours[0] = '' commands = [config['gradle']] - if 'preassemble' in thisbuild: + if thisbuild['preassemble']: commands += thisbuild['preassemble'].split() flavours_cmd = ''.join(flavours) @@ -711,7 +713,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d elif thisbuild['type'] == 'ant': logging.info("Building Ant project...") cmd = ['ant'] - if 'antcommand' in thisbuild: + if thisbuild['antcommand']: cmd += [thisbuild['antcommand']] else: cmd += ['release'] @@ -744,7 +746,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d elif thisbuild['type'] == 'gradle': basename = app['id'] dd = build_dir - if 'subdir' in thisbuild: + if thisbuild['subdir']: dd = os.path.join(dd, thisbuild['subdir']) basename = os.path.basename(thisbuild['subdir']) if '@' in thisbuild['gradle']: @@ -801,7 +803,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d elif line.startswith("native-code:"): nativecode = line[12:] - if thisbuild.get('buildjni') is not None: + if thisbuild['buildjni']: if nativecode is None or "'" not in nativecode: raise BuildException("Native code should have been built but none was packaged") if thisbuild['novcheck']: @@ -875,7 +877,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, if os.path.exists(dest_also): return False - if 'disable' in thisbuild: + if thisbuild['disable']: return False logging.info("Building version " + thisbuild['version'] + ' of ' + app['id']) @@ -997,7 +999,7 @@ def main(): if options.latest: for app in apps: for build in reversed(app['builds']): - if 'disable' in build: + if build['disable']: continue app['builds'] = [build] break diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index c9405ad3..ae185350 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -109,9 +109,9 @@ def check_tags(app, pattern): flavour = None if len(app['builds']) > 0: - if 'subdir' in app['builds'][-1]: + if app['builds'][-1]['subdir']: build_dir = os.path.join(build_dir, app['builds'][-1]['subdir']) - if 'gradle' in app['builds'][-1]: + if app['builds'][-1]['gradle']: flavour = app['builds'][-1]['gradle'] if flavour == 'yes': flavour = None @@ -198,9 +198,9 @@ def check_repomanifest(app, branch=None): flavour = None if len(app['builds']) > 0: - if 'subdir' in app['builds'][-1]: + if app['builds'][-1]['subdir']: build_dir = os.path.join(build_dir, app['builds'][-1]['subdir']) - if 'gradle' in app['builds'][-1]: + if app['builds'][-1]['gradle']: flavour = app['builds'][-1]['gradle'] if flavour == 'yes': flavour = None @@ -434,9 +434,9 @@ def main(): flavour = None if len(app['builds']) > 0: - if 'subdir' in app['builds'][-1]: + if app['builds'][-1]['subdir']: app_dir = os.path.join(app_dir, app['builds'][-1]['subdir']) - if 'gradle' in app['builds'][-1]: + if app['builds'][-1]['gradle']: flavour = app['builds'][-1]['gradle'] if flavour == 'yes': flavour = None @@ -491,9 +491,8 @@ def main(): if not gotcur: newbuild = latest.copy() - for k in ('origlines', 'disable'): - if k in newbuild: - del newbuild[k] + del newbuild['origlines'] + newbuild['disable'] = False newbuild['vercode'] = app['Current Version Code'] newbuild['version'] = app['Current Version'] + suffix logging.info("...auto-generating build for " + newbuild['version']) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 980a3837..9854cd32 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1018,7 +1018,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False): # Optionally, the actual app source can be in a subdirectory - if 'subdir' in build: + if build['subdir']: root_dir = os.path.join(build_dir, build['subdir']) else: root_dir = build_dir @@ -1038,7 +1038,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= raise BuildException('Missing subdir ' + root_dir) # Run an init command if one is required - if 'init' in build: + if build['init']: cmd = replace_config_vars(build['init']) logging.info("Running 'init' commands in %s" % root_dir) @@ -1048,7 +1048,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= (app['id'], build['version']), p.stdout) # Apply patches if any - if 'patch' in build: + if build['patch']: + logging.info("Applying patches") for patch in build['patch']: patch = patch.strip() logging.info("Applying " + patch) @@ -1059,7 +1060,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Get required source libraries srclibpaths = [] - if 'srclibs' in build: + if build['srclibs']: logging.info("Collecting source libraries") for lib in build['srclibs']: srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths, @@ -1075,7 +1076,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Update the local.properties file localprops = [os.path.join(build_dir, 'local.properties')] - if 'subdir' in build: + if build['subdir']: localprops += [os.path.join(root_dir, 'local.properties')] for path in localprops: if not os.path.isfile(path): @@ -1099,7 +1100,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= props += "ndk.dir=%s\n" % config['ndk_path'] props += "ndk-location=%s\n" % config['ndk_path'] # Add java.encoding if necessary - if 'encoding' in build: + if build['encoding']: props += "java.encoding=%s\n" % build['encoding'] f = open(path, 'w') f.write(props) @@ -1111,7 +1112,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= if flavour in ['main', 'yes', '']: flavour = None - if 'target' in build: + if build['target']: n = build["target"].split('-')[1] FDroidPopen(['sed', '-i', 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g', @@ -1169,7 +1170,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= raise BuildException("Failed to amend build.gradle") # Delete unwanted files - if 'rm' in build: + if build['rm']: + logging.info("Removing specified files") for part in getpaths(build_dir, build, 'rm'): dest = os.path.join(build_dir, part) logging.info("Removing {0}".format(part)) @@ -1184,7 +1186,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= remove_signing_keys(build_dir) # Add required external libraries - if 'extlibs' in build: + if build['extlibs']: logging.info("Collecting prebuilt libraries") libsdir = os.path.join(root_dir, 'libs') if not os.path.exists(libsdir): @@ -1199,7 +1201,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= shutil.copyfile(libsrc, os.path.join(libsdir, libf)) # Run a pre-build command if one is required - if 'prebuild' in build: + if build['prebuild']: + logging.info("Running 'prebuild' commands in %s" % root_dir) + cmd = replace_config_vars(build['prebuild']) # Substitute source library paths into prebuild commands @@ -1207,27 +1211,24 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) - logging.info("Running 'prebuild' commands in %s" % root_dir) - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running prebuild command for %s:%s" % (app['id'], build['version']), p.stdout) - updatemode = build.get('update', ['auto']) # Generate (or update) the ant build file, build.xml... - if updatemode != ['no'] and build['type'] == 'ant': + if build['update'] and build['update'] != ['no'] and build['type'] == 'ant': parms = [os.path.join(config['sdk_path'], 'tools', 'android'), 'update'] lparms = parms + ['lib-project'] parms = parms + ['project'] - if 'target' in build and build['target']: + if build['target']: parms += ['-t', build['target']] lparms += ['-t', build['target']] - if updatemode == ['auto']: + if build['update'] == ['auto']: update_dirs = ant_subprojects(root_dir) + ['.'] else: - update_dirs = updatemode + update_dirs = build['update'] for d in update_dirs: subdir = os.path.join(root_dir, d) @@ -1254,8 +1255,6 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Split and extend via globbing the paths from a field def getpaths(build_dir, build, field): paths = [] - if field not in build: - return paths for p in build[field]: p = p.strip() full_path = os.path.join(build_dir, p) @@ -1394,7 +1393,7 @@ def scan_source(build_dir, root_dir, thisbuild): # indicate a problem (if it's not a problem, explicitly use # buildjni=no to bypass this check) if (os.path.exists(os.path.join(root_dir, 'jni')) and - thisbuild.get('buildjni') is None): + not thisbuild['buildjni']): logging.warn('Found jni directory, but buildjni is not enabled') count += 1 diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 5499d6e7..c73bb459 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -170,7 +170,7 @@ def main(): continue for build in app['builds']: - if 'commit' in build and 'disable' not in build: + if build['commit'] and not build['disable']: lastcommit = build['commit'] # Potentially incorrect UCM @@ -225,16 +225,14 @@ def main(): # Build warnings for build in app['builds']: for n in ['master', 'origin/', 'default', 'trunk']: - if 'commit' in build: - if build['commit'].startswith(n): - warn("Branch '%s' used as commit in build '%s'" % ( - n, build['version'])) - if 'srclibs' in build: - for srclib in build['srclibs']: - ref = srclib.split('@')[1].split('/')[0] - if ref.startswith(n): - warn("Branch '%s' used as commit in srclib '%s'" % ( - n, srclib)) + if build['commit'] and build['commit'].startswith(n): + warn("Branch '%s' used as commit in build '%s'" % ( + n, build['version'])) + for srclib in build['srclibs']: + ref = srclib.split('@')[1].split('/')[0] + if ref.startswith(n): + warn("Branch '%s' used as commit in srclib '%s'" % ( + n, srclib)) if not appid: print diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 67081de8..ef302721 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -23,6 +23,8 @@ import glob import cgi import logging +from collections import OrderedDict + srclibs = {} @@ -33,51 +35,73 @@ class MetaDataException(Exception): def __str__(self): return self.value -app_defaults = { - 'Disabled': None, - 'AntiFeatures': None, - 'Provides': None, - 'Categories': ['None'], - 'License': 'Unknown', - 'Web Site': '', - 'Source Code': '', - 'Issue Tracker': '', - 'Donate': None, - 'FlattrID': None, - 'Bitcoin': None, - 'Litecoin': None, - 'Dogecoin': None, - 'Name': None, - 'Auto Name': '', - 'Summary': '', - 'Description': [], - 'Requires Root': False, - 'Repo Type': '', - 'Repo': '', - 'Maintainer Notes': [], - 'Archive Policy': None, - 'Auto Update Mode': 'None', - 'Update Check Mode': 'None', - 'Update Check Ignore': None, - 'Vercode Operation': None, - 'Update Check Name': None, - 'Update Check Data': None, - 'Current Version': '', - 'Current Version Code': '0', - 'No Source Since': '' - } +# In the order in which they are laid out on files +app_defaults = OrderedDict([ + ('Disabled', None), + ('AntiFeatures', None), + ('Provides', None), + ('Categories', ['None']), + ('License', 'Unknown'), + ('Web Site', ''), + ('Source Code', ''), + ('Issue Tracker', ''), + ('Donate', None), + ('FlattrID', None), + ('Bitcoin', None), + ('Litecoin', None), + ('Dogecoin', None), + ('Name', None), + ('Auto Name', ''), + ('Summary', ''), + ('Description', []), + ('Requires Root', False), + ('Repo Type', ''), + ('Repo', ''), + ('Maintainer Notes', []), + ('Archive Policy', None), + ('Auto Update Mode', 'None'), + ('Update Check Mode', 'None'), + ('Update Check Ignore', None), + ('Vercode Operation', None), + ('Update Check Name', None), + ('Update Check Data', None), + ('Current Version', ''), + ('Current Version Code', '0'), + ('No Source Since', ''), + ]) -# This defines the preferred order for the build items - as in the -# manual, they're roughly in order of application. -ordered_flags = [ - 'disable', 'commit', 'subdir', 'submodules', 'init', - 'gradle', 'maven', 'kivy', 'output', 'oldsdkloc', 'target', - 'update', 'encoding', 'forceversion', 'forcevercode', 'rm', - 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore', - 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir', - 'antcommand', 'novcheck' - ] +# In the order in which they are laid out on files +# Sorted by their action and their place in the build timeline +flag_defaults = OrderedDict([ + ('disable', False), + ('commit', None), + ('subdir', None), + ('submodules', False), + ('init', None), + ('patch', []), + ('gradle', False), + ('maven', False), + ('kivy', False), + ('output', None), + ('srclibs', []), + ('oldsdkloc', False), + ('encoding', None), + ('forceversion', False), + ('forcevercode', False), + ('rm', []), + ('extlibs', []), + ('prebuild', []), + ('update', ['auto']), + ('target', None), + ('scanignore', []), + ('scandelete', []), + ('build', []), + ('buildjni', []), + ('preassemble', []), + ('antcommand', None), + ('novcheck', False), + ]) # Designates a metadata field type and checks that it matches @@ -198,7 +222,7 @@ def check_metadata(info): v.check(info[field], info['id']) for build in info['builds']: for attr in v.attrs: - if attr in build: + if build[attr]: v.check(build[attr], info['id']) @@ -496,7 +520,7 @@ def flagtype(name): if name in ['init', 'prebuild', 'build']: return 'script' if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode', - 'novcheck']: + 'novcheck']: return 'bool' return 'string' @@ -541,7 +565,7 @@ def parse_metadata(metafile): .format(pk, thisbuild['version'], linedesc)) pk = pk.lstrip() - if pk not in ordered_flags: + if pk not in flag_defaults: raise MetaDataException("Unrecognised build flag at {0} in {1}" .format(p, linedesc)) t = flagtype(pk) @@ -598,9 +622,9 @@ def parse_metadata(metafile): def get_build_type(build): for t in ['maven', 'gradle', 'kivy']: - if build.get(t, 'no') != 'no': + if build[t]: return t - if 'output' in build: + if build['output']: return 'raw' return 'ant' @@ -626,16 +650,6 @@ def parse_metadata(metafile): curcomments = [] curbuild = None - def fill_bool_defaults(build): - # TODO: quick fix to make bool flags default to False - # Should provide defaults for all flags instead of using - # build.get(flagname, default) each time - for f in ordered_flags: - if f in build: - continue - if flagtype(f) == 'bool': - build[f] = False - c = 0 for line in metafile: c += 1 @@ -647,7 +661,6 @@ def parse_metadata(metafile): raise MetaDataException("No commit specified for {0} in {1}" .format(curbuild['version'], linedesc)) - fill_bool_defaults(curbuild) thisinfo['builds'].append(curbuild) add_comments('build:' + curbuild['version']) mode = 0 @@ -696,7 +709,7 @@ def parse_metadata(metafile): mode = 2 buildlines = [value[:-1]] else: - thisinfo['builds'].append(parse_buildline([value])) + curbuild = parse_buildline([value]) add_comments('build:' + thisinfo['builds'][-1]['version']) elif fieldtype == 'buildv2': curbuild = {} @@ -723,7 +736,6 @@ def parse_metadata(metafile): else: buildlines.append(line) curbuild = parse_buildline(buildlines) - fill_bool_defaults(curbuild) thisinfo['builds'].append(curbuild) add_comments('build:' + thisinfo['builds'][-1]['version']) mode = 0 @@ -741,6 +753,10 @@ def parse_metadata(metafile): thisinfo['Description'].append('No description available') for build in thisinfo['builds']: + for flag, value in flag_defaults.iteritems(): + if flag in build: + continue + build[flag] = value build['type'] = get_build_type(build) return thisinfo @@ -819,10 +835,11 @@ def write_metadata(dest, app): if key in ['version', 'vercode', 'origlines', 'type']: return - t = flagtype(key) - if t == 'bool' and value == False: + if value == flag_defaults[key]: return + t = flagtype(key) + logging.debug("...writing {0} : {1}".format(key, value)) outline = ' %s=' % key @@ -838,9 +855,10 @@ def write_metadata(dest, app): outline += '\n' mf.write(outline) - for key in ordered_flags: - if key in build: - write_builditem(key, build[key]) + for flag in flag_defaults: + value = build[flag] + if value: + write_builditem(flag, value) mf.write('\n') if app['Maintainer Notes']: diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 6d0ea7ca..6918c33c 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -83,7 +83,7 @@ def main(): for thisbuild in app['builds']: - if 'disable' in thisbuild: + if thisbuild['disable']: logging.info("...skipping version %s - %s" % ( thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:]))) else: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index facd82da..c2679ca8 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -126,7 +126,7 @@ def update_wiki(apps, apks): apklist.append(apk) # Include ones we can't build, as a special case... for thisbuild in app['builds']: - if 'disable' in thisbuild: + if thisbuild['disable']: if thisbuild['vercode'] == app['Current Version Code']: cantupdate = True # TODO: Nasty: vercode is a string in the build, and an int elsewhere @@ -272,7 +272,7 @@ def delete_disabled_builds(apps, apkcache, repodirs): """ for app in apps: for build in app['builds']: - if 'disable' in build: + if build['disable']: apkfilename = app['id'] + '_' + str(build['vercode']) + '.apk' for repodir in repodirs: apkpath = os.path.join(repodir, apkfilename)