diff --git a/fdroid b/fdroid index 0213083e..aebcf3b0 100755 --- a/fdroid +++ b/fdroid @@ -52,7 +52,7 @@ def main(): sys.exit(1) if not os.path.isfile('config.py'): - print "ERROR: No config.py configuration file present" + print "Missing config file - is this a repo directory?" sys.exit(2) for basedir in ('metadata', 'tmp'): diff --git a/fdroidserver/build.py b/fdroidserver/build.py index e5ccd19b..d992a260 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -326,8 +326,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force): print "Suspending build server" subprocess.call(['vagrant', 'suspend'], cwd='builder') -def adapt_gradle(path, verbose): - if verbose: +def adapt_gradle(path): + if options.verbose: print "Adapting build.gradle at %s" % path subprocess.call(['sed', '-i', @@ -337,13 +337,13 @@ def adapt_gradle(path, verbose): 's@com.android.tools.build:gradle:[0-9\.\+]*@com.android.tools.build:gradle:'+ config['gradle_plugin'] +'@g', path]) -def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver): +def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver): """Do a build locally.""" # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, config['sdk_path'], config['ndk_path'], - config['javacc_path'], config['mvn3'], verbose, onserver) + config['javacc_path'], config['mvn3'], onserver) # We need to clean via the build tool in case the binary dirs are # different from the default ones @@ -357,7 +357,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d else: maven_dir = root_dir - p = FDroidPopen(cmd, cwd=maven_dir, verbose=verbose) + p = FDroidPopen(cmd, cwd=maven_dir) elif 'gradle' in thisbuild: print "Cleaning Gradle project..." cmd = [config['gradle'], 'clean'] @@ -367,11 +367,11 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d else: gradle_dir = root_dir - p = FDroidPopen(cmd, cwd=gradle_dir, verbose=verbose) + p = FDroidPopen(cmd, cwd=gradle_dir) elif thisbuild.get('update', '.') != 'no': print "Cleaning Ant project..." cmd = ['ant', 'clean'] - p = FDroidPopen(cmd, cwd=root_dir, verbose=verbose) + p = FDroidPopen(cmd, cwd=root_dir) if p is not None and p.returncode != 0: raise BuildException("Error cleaning %s:%s" % @@ -420,11 +420,11 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d cmd = cmd.replace('$$SDK$$', config['sdk_path']) cmd = cmd.replace('$$NDK$$', config['ndk_path']) cmd = cmd.replace('$$MVN3$$', config['mvn3']) - if verbose: + if options.verbose: print "Running 'build' commands in %s" % root_dir p = FDroidPopen(['bash', '-x', '-c', cmd], - cwd=root_dir, verbose=verbose) + cwd=root_dir) if p.returncode != 0: raise BuildException("Error running build command for %s:%s" % @@ -453,8 +453,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text - p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d), - verbose=verbose) + p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d)) if p.returncode != 0: raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) @@ -486,7 +485,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if 'mvnflags' in thisbuild: mvncmd += thisbuild['mvnflags'] - p = FDroidPopen(mvncmd, cwd=maven_dir, verbose=verbose, apkoutput=True) + p = FDroidPopen(mvncmd, cwd=maven_dir, apkoutput=True) elif 'gradle' in thisbuild: print "Building Gradle project..." @@ -512,7 +511,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d for root, dirs, files in os.walk(build_dir): for f in files: if f == 'build.gradle': - adapt_gradle(os.path.join(root, f), verbose) + adapt_gradle(os.path.join(root, f)) break if flavour in ['main', 'yes', '']: @@ -527,7 +526,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d else: commands += ['assemble'+flavour+'Release'] - p = FDroidPopen(commands, cwd=gradle_dir, verbose=verbose) + p = FDroidPopen(commands, cwd=gradle_dir) else: print "Building Ant project..." @@ -538,7 +537,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d cmd += [thisbuild['antcommand']] else: cmd += ['release'] - p = FDroidPopen(cmd, cwd=root_dir, verbose=verbose, apkoutput=True) + p = FDroidPopen(cmd, cwd=root_dir, apkoutput=True) if p.returncode != 0: raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) @@ -647,7 +646,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, - tmp_dir, repo_dir, vcs, test, server, install, force, verbose, onserver): + tmp_dir, repo_dir, vcs, test, server, install, force, onserver): """ Build a particular version of an application, if it needs building. @@ -691,7 +690,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, build_server(app, thisbuild, vcs, build_dir, output_dir, config['sdk_path'], force) else: - build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver) + build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver) return True @@ -754,16 +753,15 @@ def parse_commandline(): return options, args options = None -config = {} +config = None def main(): - global options - - # Read configuration... - common.read_config(config) + global options, config options, args = parse_commandline() + config = common.read_config(options) + if config['build_server_always']: options.server = True if options.resetserver and not options.server: @@ -771,7 +769,7 @@ def main(): sys.exit(1) # Get all apps... - apps = common.read_metadata(options.verbose, xref=not options.onserver) + apps = common.read_metadata(xref=not options.onserver) log_dir = 'logs' if not os.path.isdir(log_dir): @@ -863,8 +861,7 @@ def main(): print "Checking " + thisbuild['version'] if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, - options.server, options.install, options.force, - options.verbose, options.onserver): + options.server, options.install, options.force, options.onserver): build_succeeded.append(app) wikilog = "Build succeeded" except BuildException as be: diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 97bad1d0..12de4e46 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -268,12 +268,12 @@ def check_gplay(app): return (version.strip(), None) -config = {} +config = None +options = None def main(): - # Read configuration... - common.read_config(config) + global config, options # Parse command line... parser = OptionParser() @@ -291,6 +291,8 @@ def main(): help="Only print differences with the Play Store") (options, args) = parser.parse_args() + config = common.read_config(options) + # Get all apps... apps = common.read_metadata(options.verbose) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 024e3dda..a41fa0a4 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -27,24 +27,38 @@ import Queue import threading import magic -def read_config(config): +config = None +options = None + +def read_config(opts): """Read the repository config The config is read from config.py, which is in the current directory when any of the repo management commands are used. """ + global config, options + + if config is not None: + return config if not os.path.isfile('config.py'): print "Missing config file - is this a repo directory?" sys.exit(2) - config['build_server_always'] = False - config['mvn3'] = "mvn3" - config['archive_older'] = 0 - config['gradle'] = 'gradle' - config['update_stats'] = False - config['archive_older'] = 0 - config['max_icon_size'] = 72 - config['stats_to_carbon'] = False + + options = opts + + config = { + 'build_server_always': False, + 'mvn3': "mvn3", + 'archive_older': 0, + 'gradle': 'gradle', + 'update_stats': False, + 'archive_older': 0, + 'max_icon_size': 72, + 'stats_to_carbon': False + } + print "Reading config.py..." execfile('config.py', config) + return config def getvcs(vcstype, remote, local, sdk_path): @@ -491,7 +505,7 @@ def metafieldtype(name): # 'descriptionlines' - original lines of description as formatted in the # metadata file. # -def parse_metadata(metafile, verbose=False): +def parse_metadata(metafile): def parse_buildline(lines): value = "".join(lines) @@ -730,7 +744,7 @@ def getsrcname(app, build): # # 'dest' - The path to the output file # 'app' - The app data -def write_metadata(dest, app, verbose=False): +def write_metadata(dest, app): def writecomments(key): written = 0 @@ -738,7 +752,7 @@ def write_metadata(dest, app, verbose=False): if pf == key: mf.write(comment + '\n') written += 1 - if verbose and written > 0: + if options.verbose and written > 0: print "...writing comments for " + (key if key else 'EOF') def writefield(field, value=None): @@ -802,7 +816,7 @@ def write_metadata(dest, app, verbose=False): def write_builditem(key, value): if key not in ['version', 'vercode', 'origlines']: - if verbose: + if options.verbose: print "...writing {0} : {1}".format(key, value) outline = ' ' + key + '=' bits = value.split('&& ') @@ -836,12 +850,12 @@ def write_metadata(dest, app, verbose=False): # Read all metadata. Returns a list of 'app' objects (which are dictionaries as # returned by the parse_metadata function. -def read_metadata(verbose=False, xref=True, package=None): +def read_metadata(xref=True, package=None): apps = [] for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))): if package is None or metafile == os.path.join('metadata', package + '.txt'): try: - appinfo = parse_metadata(metafile, verbose=verbose) + appinfo = parse_metadata(metafile) except Exception, e: raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e))) apps.append(appinfo) @@ -1308,12 +1322,11 @@ def getsrclib(spec, srclib_dir, sdk_path, ndk_path="", mvn3="", basepath=False, # 'ndk_path' - the path to the Android NDK # 'javacc_path' - the path to javacc # 'mvn3' - the path to the maven 3 executable -# 'verbose' - optional: verbose or not (default=False) # Returns the (root, srclibpaths) where: # 'root' is the root directory, which may be the same as 'build_dir' or may # be a subdirectory of it. # 'srclibpaths' is information on the srclibs being used -def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, verbose=False, onserver=False): +def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, onserver=False): # Optionally, the actual app source can be in a subdirectory... if 'subdir' in build: @@ -1333,6 +1346,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, # Initialise submodules if requred... if build.get('submodules', 'no') == 'yes': if verbose: print "Initialising submodules..." + if build.get('submodules', 'no') == 'yes': + if options.verbose: + print "Initialising submodules..." vcs.initsubmodules() # Run an init command if one is required... @@ -1341,10 +1357,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, cmd = cmd.replace('$$SDK$$', sdk_path) cmd = cmd.replace('$$NDK$$', ndk_path) cmd = cmd.replace('$$MVN$$', mvn3) - if verbose: + if options.verbose: print "Running 'init' commands in %s" % root_dir - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir, verbose=verbose) + p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running init command for %s:%s" % (app['id'], build['version']), p.stdout, p.stderr) @@ -1380,10 +1396,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, if os.path.exists(badpath): print "Removing '%s'" % badpath shutil.rmtree(badpath) - if verbose: + if options.verbose: print "Update of '%s': exec '%s' in '%s'"%\ (d," ".join(parms),cwd) - p = FDroidPopen(parms, cwd=cwd, verbose=verbose) + p = FDroidPopen(parms, cwd=cwd) # check to see whether an error was returned without a proper exit code (this is the case for the 'no target set or target invalid' error) if p.returncode != 0 or (p.stderr != "" and p.stderr.startswith("Error: ")): raise BuildException("Failed to update project at %s" % cwd, @@ -1399,7 +1415,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, for root, dirs, files in os.walk(build_dir): for f in files: if f == 'build.gradle': - clean_gradle_keys(os.path.join(root, f), verbose) + clean_gradle_keys(os.path.join(root, f)) break # Update the local.properties file... @@ -1564,10 +1580,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, cmd = cmd.replace('$$SDK$$', sdk_path) cmd = cmd.replace('$$NDK$$', ndk_path) cmd = cmd.replace('$$MVN3$$', mvn3) - if verbose: + if options.verbose: print "Running 'prebuild' commands in %s" % root_dir - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir, verbose=verbose) + p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running prebuild command for %s:%s" % (app['id'], build['version']), p.stdout, p.stderr) @@ -1805,15 +1821,14 @@ class PopenResult: def FDroidPopen(commands, cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - verbose=False, apkoutput=False): + apkoutput=False): """ Runs a command the FDroid way and returns return code and output :param commands, cwd, stdout, stderr: like subprocess.Popen - :param verbose: whether to print output as it is saved """ - if verbose: + if options.verbose: print "Directory: %s" % cwd print " > %s" % ' '.join(commands) @@ -1832,7 +1847,7 @@ def FDroidPopen(commands, cwd, # Show what we received from standard output while not stdout_queue.empty(): line = stdout_queue.get() - if verbose: + if options.verbose: # Output directly to console sys.stdout.write(line) sys.stdout.flush() @@ -1843,7 +1858,7 @@ def FDroidPopen(commands, cwd, # Show what we received from standard error while not stderr_queue.empty(): line = stderr_queue.get() - if verbose: + if options.verbose: # Output directly to console sys.stderr.write(line) sys.stderr.flush() @@ -1854,8 +1869,8 @@ def FDroidPopen(commands, cwd, result.returncode = p.returncode return result -def clean_gradle_keys(path, verbose): - if verbose: +def clean_gradle_keys(path): + if options.verbose: print "Cleaning build.gradle of keysigning configs at %s" % path lines = None diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 7ae19dfe..2c1ff651 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -82,14 +82,12 @@ def getrepofrompage(url): return (None, "No information found." + page) -config = {} +config = None +options = None def main(): - # Read configuration... - common.read_config(config) - - import common + global config, options # Parse command line... parser = OptionParser() @@ -103,6 +101,8 @@ def main(): help="Allows a different revision (or git branch) to be specified for the initial import") (options, args) = parser.parse_args() + config = common.read_config(options) + if not options.url: print "Specify project url." sys.exit(1) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 6eccabc8..9ec10a7f 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -29,12 +29,12 @@ from optparse import OptionParser import common from common import BuildException -config = {} +config = None +options = None def main(): - # Read configuration... - common.read_config(config) + global config, options # Parse command line... parser = OptionParser() @@ -44,6 +44,8 @@ def main(): help="Publish only the specified package") (options, args) = parser.parse_args() + config = common.read_config(options) + log_dir = 'logs' if not os.path.isdir(log_dir): print "Creating log directory" diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 11fee08b..d28dc936 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -22,10 +22,12 @@ import os from optparse import OptionParser import common +config = None +options = None + def main(): - #Read configuration... - execfile('config.py', globals()) + global config, options # Parse command line... parser = OptionParser() @@ -35,8 +37,10 @@ def main(): help="Process only the specified package") (options, args) = parser.parse_args() + config = common.read_config(options) + # Get all apps... - apps = common.read_metadata(options.verbose, package=options.package) + apps = common.read_metadata(package=options.package) if len(apps) == 0 and options.package: print "No such package" @@ -44,7 +48,7 @@ def main(): for app in apps: print "Writing " + app['id'] - common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app, verbose=options.verbose) + common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) print "Finished." diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index e571c32d..745010f3 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -25,13 +25,12 @@ import common from common import BuildException from common import VCSException -config = {} +config = None +options = None def main(): - # Read configuration... - common.read_config(config) - + global config, options # Parse command line... parser = OptionParser() @@ -43,8 +42,10 @@ def main(): help="Skip svn repositories - for test purposes, because they are too slow.") (options, args) = parser.parse_args() + config = common.read_config(options) + # Get all apps... - apps = common.read_metadata(options.verbose) + apps = common.read_metadata() # Filter apps according to command-line options if options.package: @@ -99,7 +100,7 @@ def main(): build_dir, srclib_dir, extlib_dir, config['sdk_path'], config['ndk_path'], config['javacc_path'], config['mvn3'], - options.verbose, False) + False) # Do the scan... buildprobs = common.scan_source(build_dir, root_dir, thisbuild) diff --git a/fdroidserver/server.py b/fdroidserver/server.py index baa9dece..bfe5e5be 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -23,12 +23,14 @@ import subprocess from optparse import OptionParser import common -config = {} +config = None +options = None def main(): - # Read configuration... - common.read_config(config) + global config, options + + config = common.read_config(options) # Parse command line... parser = OptionParser() diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py index ad5b82c0..661da4e1 100644 --- a/fdroidserver/stats.py +++ b/fdroidserver/stats.py @@ -36,12 +36,12 @@ def carbon_send(key, value): s.sendall(msg) s.close() -config = {} +options = None +config = None def main(): - # Read configuration... - common.read_config(config) + global options, config if not config['update_stats']: print "Stats are disabled - check your configuration" @@ -55,6 +55,8 @@ def main(): help="Download logs we don't have") (options, args) = parser.parse_args() + config = common.read_config(options) + # Get all metadata-defined apps... metaapps = common.read_metadata(options.verbose) @@ -114,7 +116,6 @@ def main(): for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')): if options.verbose: print '...' + logfile - logdate = logfile[len(logsdir) + 1 + len('access-'):-7] p = subprocess.Popen(["zcat", logfile], stdout = subprocess.PIPE) matches = (logsearch(line) for line in p.stdout) for match in matches: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 16fe6008..ac7565a4 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -34,12 +34,11 @@ import common from common import MetaDataException from PIL import Image -def update_wiki(apps, apks, verbose=False): +def update_wiki(apps, apks): """Update the wiki :param apps: fully populated list of all applications :param apks: all apks, except... - :param verbose: True to make a lot of noise """ print "Updating wiki" wikicat = 'Apps' @@ -205,13 +204,13 @@ def update_wiki(apps, apks, verbose=False): print "Updating modified page " + page.name page.save(genp[page.name], summary='Auto-updated') else: - if verbose: + if options.verbose: print "Page " + page.name + " is unchanged" else: print "Deleting page " + page.name page.delete('No longer published') for pagename, text in genp.items(): - if verbose: + if options.verbose: print "Checking " + pagename if not pagename in existingpages: print "Creating page " + pagename @@ -682,16 +681,14 @@ def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions): apks.remove(apk) -config = {} +config = None options = None def main(): - # Read configuration... - common.read_config(config) + global config, options # Parse command line... - global options parser = OptionParser() parser.add_option("-c", "--createmeta", action="store_true", default=False, help="Create skeleton metadata files that are missing") @@ -716,6 +713,8 @@ def main(): help="Clean update - don't uses caches, reprocess all apks") (options, args) = parser.parse_args() + config = common.read_config(options) + repodirs = ['repo'] if config['archive_older'] != 0: repodirs.append('archive') @@ -727,7 +726,7 @@ def main(): sys.exit(0) # Get all apps... - apps = common.read_metadata(verbose=options.verbose) + apps = common.read_metadata() # Generate a list of categories... categories = [] @@ -869,7 +868,7 @@ def main(): if options.wiki: if archapks: apks.extend(archapks) - update_wiki(apps, apks, options.verbose) + update_wiki(apps, apks) print "Finished." diff --git a/fdroidserver/verify.py b/fdroidserver/verify.py index 5d3f4222..d81d4073 100644 --- a/fdroidserver/verify.py +++ b/fdroidserver/verify.py @@ -1,8 +1,8 @@ #!/usr/bin/env python2 # -*- coding: utf-8 -*- # -# publish.py - part of the FDroid server tools -# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com +# verify.py - part of the FDroid server tools +# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -24,12 +24,17 @@ import subprocess import glob from optparse import OptionParser +import common from common import BuildException +options = None +config = None + def main(): - #Read configuration... - execfile('config.py', globals()) + global options, config + + options, args = parse_commandline() # Parse command line... parser = OptionParser() @@ -39,6 +44,8 @@ def main(): help="Verify only the specified package") (options, args) = parser.parse_args() + config = common.read_config(options) + tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): print "Creating temporary directory"