1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-10-03 17:50:11 +02:00

Start rewriting options and config as common.py globals

This commit is contained in:
Daniel Martí 2013-11-01 12:10:57 +01:00
parent 1d88ba5450
commit c3be06fb1d
12 changed files with 129 additions and 99 deletions

2
fdroid
View File

@ -52,7 +52,7 @@ def main():
sys.exit(1)
if not os.path.isfile('config.py'):
print "ERROR: No config.py configuration file present"
print "Missing config file - is this a repo directory?"
sys.exit(2)
for basedir in ('metadata', 'tmp'):

View File

@ -326,8 +326,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force):
print "Suspending build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
def adapt_gradle(path, verbose):
if verbose:
def adapt_gradle(path):
if options.verbose:
print "Adapting build.gradle at %s" % path
subprocess.call(['sed', '-i',
@ -337,13 +337,13 @@ def adapt_gradle(path, verbose):
's@com.android.tools.build:gradle:[0-9\.\+]*@com.android.tools.build:gradle:'+ config['gradle_plugin'] +'@g', path])
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver):
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver):
"""Do a build locally."""
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, config['sdk_path'], config['ndk_path'],
config['javacc_path'], config['mvn3'], verbose, onserver)
config['javacc_path'], config['mvn3'], onserver)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
@ -357,7 +357,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir, verbose=verbose)
p = FDroidPopen(cmd, cwd=maven_dir)
elif 'gradle' in thisbuild:
print "Cleaning Gradle project..."
cmd = [config['gradle'], 'clean']
@ -367,11 +367,11 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
else:
gradle_dir = root_dir
p = FDroidPopen(cmd, cwd=gradle_dir, verbose=verbose)
p = FDroidPopen(cmd, cwd=gradle_dir)
elif thisbuild.get('update', '.') != 'no':
print "Cleaning Ant project..."
cmd = ['ant', 'clean']
p = FDroidPopen(cmd, cwd=root_dir, verbose=verbose)
p = FDroidPopen(cmd, cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
@ -420,11 +420,11 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
cmd = cmd.replace('$$NDK$$', config['ndk_path'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
if verbose:
if options.verbose:
print "Running 'build' commands in %s" % root_dir
p = FDroidPopen(['bash', '-x', '-c', cmd],
cwd=root_dir, verbose=verbose)
cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
@ -453,8 +453,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d),
verbose=verbose)
p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
@ -486,7 +485,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if 'mvnflags' in thisbuild:
mvncmd += thisbuild['mvnflags']
p = FDroidPopen(mvncmd, cwd=maven_dir, verbose=verbose, apkoutput=True)
p = FDroidPopen(mvncmd, cwd=maven_dir, apkoutput=True)
elif 'gradle' in thisbuild:
print "Building Gradle project..."
@ -512,7 +511,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
for root, dirs, files in os.walk(build_dir):
for f in files:
if f == 'build.gradle':
adapt_gradle(os.path.join(root, f), verbose)
adapt_gradle(os.path.join(root, f))
break
if flavour in ['main', 'yes', '']:
@ -527,7 +526,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
else:
commands += ['assemble'+flavour+'Release']
p = FDroidPopen(commands, cwd=gradle_dir, verbose=verbose)
p = FDroidPopen(commands, cwd=gradle_dir)
else:
print "Building Ant project..."
@ -538,7 +537,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
cmd += [thisbuild['antcommand']]
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir, verbose=verbose, apkoutput=True)
p = FDroidPopen(cmd, cwd=root_dir, apkoutput=True)
if p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
@ -647,7 +646,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, test, server, install, force, verbose, onserver):
tmp_dir, repo_dir, vcs, test, server, install, force, onserver):
"""
Build a particular version of an application, if it needs building.
@ -691,7 +690,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
build_server(app, thisbuild, vcs, build_dir, output_dir, config['sdk_path'], force)
else:
build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver)
build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver)
return True
@ -754,16 +753,15 @@ def parse_commandline():
return options, args
options = None
config = {}
config = None
def main():
global options
# Read configuration...
common.read_config(config)
global options, config
options, args = parse_commandline()
config = common.read_config(options)
if config['build_server_always']:
options.server = True
if options.resetserver and not options.server:
@ -771,7 +769,7 @@ def main():
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose, xref=not options.onserver)
apps = common.read_metadata(xref=not options.onserver)
log_dir = 'logs'
if not os.path.isdir(log_dir):
@ -863,8 +861,7 @@ def main():
print "Checking " + thisbuild['version']
if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir,
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test,
options.server, options.install, options.force,
options.verbose, options.onserver):
options.server, options.install, options.force, options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:

View File

@ -268,12 +268,12 @@ def check_gplay(app):
return (version.strip(), None)
config = {}
config = None
options = None
def main():
# Read configuration...
common.read_config(config)
global config, options
# Parse command line...
parser = OptionParser()
@ -291,6 +291,8 @@ def main():
help="Only print differences with the Play Store")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
apps = common.read_metadata(options.verbose)

View File

@ -27,24 +27,38 @@ import Queue
import threading
import magic
def read_config(config):
config = None
options = None
def read_config(opts):
"""Read the repository config
The config is read from config.py, which is in the current directory when
any of the repo management commands are used.
"""
global config, options
if config is not None:
return config
if not os.path.isfile('config.py'):
print "Missing config file - is this a repo directory?"
sys.exit(2)
config['build_server_always'] = False
config['mvn3'] = "mvn3"
config['archive_older'] = 0
config['gradle'] = 'gradle'
config['update_stats'] = False
config['archive_older'] = 0
config['max_icon_size'] = 72
config['stats_to_carbon'] = False
options = opts
config = {
'build_server_always': False,
'mvn3': "mvn3",
'archive_older': 0,
'gradle': 'gradle',
'update_stats': False,
'archive_older': 0,
'max_icon_size': 72,
'stats_to_carbon': False
}
print "Reading config.py..."
execfile('config.py', config)
return config
def getvcs(vcstype, remote, local, sdk_path):
@ -491,7 +505,7 @@ def metafieldtype(name):
# 'descriptionlines' - original lines of description as formatted in the
# metadata file.
#
def parse_metadata(metafile, verbose=False):
def parse_metadata(metafile):
def parse_buildline(lines):
value = "".join(lines)
@ -730,7 +744,7 @@ def getsrcname(app, build):
#
# 'dest' - The path to the output file
# 'app' - The app data
def write_metadata(dest, app, verbose=False):
def write_metadata(dest, app):
def writecomments(key):
written = 0
@ -738,7 +752,7 @@ def write_metadata(dest, app, verbose=False):
if pf == key:
mf.write(comment + '\n')
written += 1
if verbose and written > 0:
if options.verbose and written > 0:
print "...writing comments for " + (key if key else 'EOF')
def writefield(field, value=None):
@ -802,7 +816,7 @@ def write_metadata(dest, app, verbose=False):
def write_builditem(key, value):
if key not in ['version', 'vercode', 'origlines']:
if verbose:
if options.verbose:
print "...writing {0} : {1}".format(key, value)
outline = ' ' + key + '='
bits = value.split('&& ')
@ -836,12 +850,12 @@ def write_metadata(dest, app, verbose=False):
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
# returned by the parse_metadata function.
def read_metadata(verbose=False, xref=True, package=None):
def read_metadata(xref=True, package=None):
apps = []
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
if package is None or metafile == os.path.join('metadata', package + '.txt'):
try:
appinfo = parse_metadata(metafile, verbose=verbose)
appinfo = parse_metadata(metafile)
except Exception, e:
raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
apps.append(appinfo)
@ -1308,12 +1322,11 @@ def getsrclib(spec, srclib_dir, sdk_path, ndk_path="", mvn3="", basepath=False,
# 'ndk_path' - the path to the Android NDK
# 'javacc_path' - the path to javacc
# 'mvn3' - the path to the maven 3 executable
# 'verbose' - optional: verbose or not (default=False)
# Returns the (root, srclibpaths) where:
# 'root' is the root directory, which may be the same as 'build_dir' or may
# be a subdirectory of it.
# 'srclibpaths' is information on the srclibs being used
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, verbose=False, onserver=False):
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, onserver=False):
# Optionally, the actual app source can be in a subdirectory...
if 'subdir' in build:
@ -1333,6 +1346,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path,
# Initialise submodules if requred...
if build.get('submodules', 'no') == 'yes':
if verbose: print "Initialising submodules..."
if build.get('submodules', 'no') == 'yes':
if options.verbose:
print "Initialising submodules..."
vcs.initsubmodules()
# Run an init command if one is required...
@ -1341,10 +1357,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path,
cmd = cmd.replace('$$SDK$$', sdk_path)
cmd = cmd.replace('$$NDK$$', ndk_path)
cmd = cmd.replace('$$MVN$$', mvn3)
if verbose:
if options.verbose:
print "Running 'init' commands in %s" % root_dir
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir, verbose=verbose)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running init command for %s:%s" %
(app['id'], build['version']), p.stdout, p.stderr)
@ -1380,10 +1396,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path,
if os.path.exists(badpath):
print "Removing '%s'" % badpath
shutil.rmtree(badpath)
if verbose:
if options.verbose:
print "Update of '%s': exec '%s' in '%s'"%\
(d," ".join(parms),cwd)
p = FDroidPopen(parms, cwd=cwd, verbose=verbose)
p = FDroidPopen(parms, cwd=cwd)
# check to see whether an error was returned without a proper exit code (this is the case for the 'no target set or target invalid' error)
if p.returncode != 0 or (p.stderr != "" and p.stderr.startswith("Error: ")):
raise BuildException("Failed to update project at %s" % cwd,
@ -1399,7 +1415,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path,
for root, dirs, files in os.walk(build_dir):
for f in files:
if f == 'build.gradle':
clean_gradle_keys(os.path.join(root, f), verbose)
clean_gradle_keys(os.path.join(root, f))
break
# Update the local.properties file...
@ -1564,10 +1580,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, sdk_path,
cmd = cmd.replace('$$SDK$$', sdk_path)
cmd = cmd.replace('$$NDK$$', ndk_path)
cmd = cmd.replace('$$MVN3$$', mvn3)
if verbose:
if options.verbose:
print "Running 'prebuild' commands in %s" % root_dir
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir, verbose=verbose)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running prebuild command for %s:%s" %
(app['id'], build['version']), p.stdout, p.stderr)
@ -1805,15 +1821,14 @@ class PopenResult:
def FDroidPopen(commands, cwd,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
verbose=False, apkoutput=False):
apkoutput=False):
"""
Runs a command the FDroid way and returns return code and output
:param commands, cwd, stdout, stderr: like subprocess.Popen
:param verbose: whether to print output as it is saved
"""
if verbose:
if options.verbose:
print "Directory: %s" % cwd
print " > %s" % ' '.join(commands)
@ -1832,7 +1847,7 @@ def FDroidPopen(commands, cwd,
# Show what we received from standard output
while not stdout_queue.empty():
line = stdout_queue.get()
if verbose:
if options.verbose:
# Output directly to console
sys.stdout.write(line)
sys.stdout.flush()
@ -1843,7 +1858,7 @@ def FDroidPopen(commands, cwd,
# Show what we received from standard error
while not stderr_queue.empty():
line = stderr_queue.get()
if verbose:
if options.verbose:
# Output directly to console
sys.stderr.write(line)
sys.stderr.flush()
@ -1854,8 +1869,8 @@ def FDroidPopen(commands, cwd,
result.returncode = p.returncode
return result
def clean_gradle_keys(path, verbose):
if verbose:
def clean_gradle_keys(path):
if options.verbose:
print "Cleaning build.gradle of keysigning configs at %s" % path
lines = None

View File

@ -82,14 +82,12 @@ def getrepofrompage(url):
return (None, "No information found." + page)
config = {}
config = None
options = None
def main():
# Read configuration...
common.read_config(config)
import common
global config, options
# Parse command line...
parser = OptionParser()
@ -103,6 +101,8 @@ def main():
help="Allows a different revision (or git branch) to be specified for the initial import")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not options.url:
print "Specify project url."
sys.exit(1)

View File

@ -29,12 +29,12 @@ from optparse import OptionParser
import common
from common import BuildException
config = {}
config = None
options = None
def main():
# Read configuration...
common.read_config(config)
global config, options
# Parse command line...
parser = OptionParser()
@ -44,6 +44,8 @@ def main():
help="Publish only the specified package")
(options, args) = parser.parse_args()
config = common.read_config(options)
log_dir = 'logs'
if not os.path.isdir(log_dir):
print "Creating log directory"

View File

@ -22,10 +22,12 @@ import os
from optparse import OptionParser
import common
config = None
options = None
def main():
#Read configuration...
execfile('config.py', globals())
global config, options
# Parse command line...
parser = OptionParser()
@ -35,8 +37,10 @@ def main():
help="Process only the specified package")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
apps = common.read_metadata(options.verbose, package=options.package)
apps = common.read_metadata(package=options.package)
if len(apps) == 0 and options.package:
print "No such package"
@ -44,7 +48,7 @@ def main():
for app in apps:
print "Writing " + app['id']
common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app, verbose=options.verbose)
common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
print "Finished."

View File

@ -25,13 +25,12 @@ import common
from common import BuildException
from common import VCSException
config = {}
config = None
options = None
def main():
# Read configuration...
common.read_config(config)
global config, options
# Parse command line...
parser = OptionParser()
@ -43,8 +42,10 @@ def main():
help="Skip svn repositories - for test purposes, because they are too slow.")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
apps = common.read_metadata(options.verbose)
apps = common.read_metadata()
# Filter apps according to command-line options
if options.package:
@ -99,7 +100,7 @@ def main():
build_dir, srclib_dir, extlib_dir,
config['sdk_path'], config['ndk_path'],
config['javacc_path'], config['mvn3'],
options.verbose, False)
False)
# Do the scan...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)

View File

@ -23,12 +23,14 @@ import subprocess
from optparse import OptionParser
import common
config = {}
config = None
options = None
def main():
# Read configuration...
common.read_config(config)
global config, options
config = common.read_config(options)
# Parse command line...
parser = OptionParser()

View File

@ -36,12 +36,12 @@ def carbon_send(key, value):
s.sendall(msg)
s.close()
config = {}
options = None
config = None
def main():
# Read configuration...
common.read_config(config)
global options, config
if not config['update_stats']:
print "Stats are disabled - check your configuration"
@ -55,6 +55,8 @@ def main():
help="Download logs we don't have")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all metadata-defined apps...
metaapps = common.read_metadata(options.verbose)
@ -114,7 +116,6 @@ def main():
for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
if options.verbose:
print '...' + logfile
logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
p = subprocess.Popen(["zcat", logfile], stdout = subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:

View File

@ -34,12 +34,11 @@ import common
from common import MetaDataException
from PIL import Image
def update_wiki(apps, apks, verbose=False):
def update_wiki(apps, apks):
"""Update the wiki
:param apps: fully populated list of all applications
:param apks: all apks, except...
:param verbose: True to make a lot of noise
"""
print "Updating wiki"
wikicat = 'Apps'
@ -205,13 +204,13 @@ def update_wiki(apps, apks, verbose=False):
print "Updating modified page " + page.name
page.save(genp[page.name], summary='Auto-updated')
else:
if verbose:
if options.verbose:
print "Page " + page.name + " is unchanged"
else:
print "Deleting page " + page.name
page.delete('No longer published')
for pagename, text in genp.items():
if verbose:
if options.verbose:
print "Checking " + pagename
if not pagename in existingpages:
print "Creating page " + pagename
@ -682,16 +681,14 @@ def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
apks.remove(apk)
config = {}
config = None
options = None
def main():
# Read configuration...
common.read_config(config)
global config, options
# Parse command line...
global options
parser = OptionParser()
parser.add_option("-c", "--createmeta", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
@ -716,6 +713,8 @@ def main():
help="Clean update - don't uses caches, reprocess all apks")
(options, args) = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
@ -727,7 +726,7 @@ def main():
sys.exit(0)
# Get all apps...
apps = common.read_metadata(verbose=options.verbose)
apps = common.read_metadata()
# Generate a list of categories...
categories = []
@ -869,7 +868,7 @@ def main():
if options.wiki:
if archapks:
apks.extend(archapks)
update_wiki(apps, apks, options.verbose)
update_wiki(apps, apks)
print "Finished."

View File

@ -1,8 +1,8 @@
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
# publish.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# verify.py - part of the FDroid server tools
# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -24,12 +24,17 @@ import subprocess
import glob
from optparse import OptionParser
import common
from common import BuildException
options = None
config = None
def main():
#Read configuration...
execfile('config.py', globals())
global options, config
options, args = parse_commandline()
# Parse command line...
parser = OptionParser()
@ -39,6 +44,8 @@ def main():
help="Verify only the specified package")
(options, args) = parser.parse_args()
config = common.read_config(options)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"