kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-14 11:00:10 +01:00
Code Activity

Handle repo config in a more sensible way

Browse Source
This commit is contained in:
Ciaran Gultnieks 2013-10-31 15:37:39 +00:00
parent 6bff3f4e77
commit 08287a1fa8
9 changed files with 145 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
fdroidserver/build.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# build.py - part of the FDroid server tools # build.py - part of the FDroid server tools
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc> # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -331,10 +331,10 @@ def adapt_gradle(path, verbose):
print "Adapting build.gradle at %s" % path print "Adapting build.gradle at %s" % path
subprocess.call(['sed', '-i', subprocess.call(['sed', '-i',
's@buildToolsVersion[ ]*["\\\'][0-9\.]*["\\\']@buildToolsVersion "'+build_tools+'"@g', path]) 's@buildToolsVersion[ ]*["\\\'][0-9\.]*["\\\']@buildToolsVersion "'+ config['build_tools'] +'"@g', path])
subprocess.call(['sed', '-i', subprocess.call(['sed', '-i',
's@com.android.tools.build:gradle:[0-9\.\+]*@com.android.tools.build:gradle:'+gradle_plugin+'@g', path]) 's@com.android.tools.build:gradle:[0-9\.\+]*@com.android.tools.build:gradle:'+ config['gradle_plugin'] +'@g', path])
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver): def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver):
@ -342,15 +342,15 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
# Prepare the source code... # Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, build_dir, srclib_dir, extlib_dir, config['sdk_path'], config['ndk_path'],
javacc_path, mvn3, verbose, onserver) config['javacc_path'], config['mvn3'], verbose, onserver)
# We need to clean via the build tool in case the binary dirs are # We need to clean via the build tool in case the binary dirs are
# different from the default ones # different from the default ones
p = None p = None
if 'maven' in thisbuild: if 'maven' in thisbuild:
print "Cleaning Maven project..." print "Cleaning Maven project..."
cmd = [mvn3, 'clean', '-Dandroid.sdk.path=' + sdk_path] cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in thisbuild['maven']: if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@')[1]) maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@')[1])
@ -360,7 +360,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
p = FDroidPopen(cmd, cwd=maven_dir, verbose=verbose) p = FDroidPopen(cmd, cwd=maven_dir, verbose=verbose)
elif 'gradle' in thisbuild: elif 'gradle' in thisbuild:
print "Cleaning Gradle project..." print "Cleaning Gradle project..."
cmd = [gradle, 'clean'] cmd = [config['gradle'], 'clean']
if '@' in thisbuild['gradle']: if '@' in thisbuild['gradle']:
gradle_dir = os.path.join(root_dir, thisbuild['gradle'].split('@')[1]) gradle_dir = os.path.join(root_dir, thisbuild['gradle'].split('@')[1])
@ -417,9 +417,9 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
for name, libpath in srclibpaths: for name, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir) libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath) cmd = cmd.replace('$$' + name + '$$', libpath)
cmd = cmd.replace('$$SDK$$', sdk_path) cmd = cmd.replace('$$SDK$$', config['sdk_path'])
cmd = cmd.replace('$$NDK$$', ndk_path) cmd = cmd.replace('$$NDK$$', config['ndk_path'])
cmd = cmd.replace('$$MVN3$$', mvn3) cmd = cmd.replace('$$MVN3$$', config['mvn3'])
if verbose: if verbose:
print "Running 'build' commands in %s" % root_dir print "Running 'build' commands in %s" % root_dir
@ -438,7 +438,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
jni_components = [''] jni_components = ['']
else: else:
jni_components = [c.strip() for c in jni_components.split(';')] jni_components = [c.strip() for c in jni_components.split(';')]
ndkbuild = os.path.join(ndk_path, "ndk-build") ndkbuild = os.path.join(config['ndk_path'], "ndk-build")
for d in jni_components: for d in jni_components:
if options.verbose: if options.verbose:
print "Running ndk-build in " + root_dir + '/' + d print "Running ndk-build in " + root_dir + '/' + d
@ -468,7 +468,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
else: else:
maven_dir = root_dir maven_dir = root_dir
mvncmd = [mvn3, '-Dandroid.sdk.path=' + sdk_path] mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path']]
if install: if install:
mvncmd += ['-Dandroid.sign.debug=true', 'package', 'android:deploy'] mvncmd += ['-Dandroid.sign.debug=true', 'package', 'android:deploy']
else: else:
@ -518,7 +518,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if flavour in ['main', 'yes', '']: if flavour in ['main', 'yes', '']:
flavour = '' flavour = ''
commands = [gradle] commands = [config['gradle']]
if 'preassemble' in thisbuild: if 'preassemble' in thisbuild:
for task in thisbuild['preassemble'].split(): for task in thisbuild['preassemble'].split():
commands.append(task) commands.append(task)
@ -583,7 +583,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
src = os.path.join(bindir, src) src = os.path.join(bindir, src)
# Make sure it's not debuggable... # Make sure it's not debuggable...
if common.isApkDebuggable(src): if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable") raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version # By way of a sanity check, make sure the version and version
@ -592,7 +592,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if not os.path.exists(src): if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src) raise BuildException("Unsigned apk is not at expected location of " + src)
p = subprocess.Popen([os.path.join(sdk_path, 'build-tools', build_tools, 'aapt'), p = subprocess.Popen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'),
'dump', 'badging', src], 'dump', 'badging', src],
stdout=subprocess.PIPE) stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]
@ -688,7 +689,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
# grabbing the source now. # grabbing the source now.
vcs.gotorevision(thisbuild['commit']) vcs.gotorevision(thisbuild['commit'])
build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force) build_server(app, thisbuild, vcs, build_dir, output_dir, config['sdk_path'], force)
else: else:
build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver) build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver)
return True return True
@ -753,19 +754,17 @@ def parse_commandline():
return options, args return options, args
options = None options = None
config = {}
def main(): def main():
global options global options
# Read configuration... # Read configuration...
globals()['build_server_always'] = False common.read_config(config)
globals()['mvn3'] = "mvn3"
globals()['archive_older'] = 0
execfile('config.py', globals())
options, args = parse_commandline() options, args = parse_commandline()
if build_server_always: if config['build_server_always']:
options.server = True options.server = True
if options.resetserver and not options.server: if options.resetserver and not options.server:
print "Using --resetserver without --server makes no sense" print "Using --resetserver without --server makes no sense"
@ -792,7 +791,7 @@ def main():
print "Creating output directory" print "Creating output directory"
os.makedirs(output_dir) os.makedirs(output_dir)
if archive_older != 0: if config['archive_older'] != 0:
also_check_dir = 'archive' also_check_dir = 'archive'
else: else:
also_check_dir = None also_check_dir = None
@ -828,8 +827,9 @@ def main():
if options.wiki: if options.wiki:
import mwclient import mwclient
site = mwclient.Site((wiki_protocol, wiki_server), path=wiki_path) site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
site.login(wiki_user, wiki_password) path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
# Build applications... # Build applications...
failed_apps = {} failed_apps = {}
@ -855,7 +855,7 @@ def main():
if options.verbose: if options.verbose:
print "Getting {0} vcs interface for {1}".format( print "Getting {0} vcs interface for {1}".format(
app['Repo Type'], app['Repo']) app['Repo Type'], app['Repo'])
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, config['sdk_path'])
first = False first = False

16
fdroidserver/checkupdates.py
View File

@ -268,11 +268,12 @@ def check_gplay(app):
return (version.strip(), None) return (version.strip(), None)
config = {}
def main(): def main():
# Read configuration... # Read configuration...
globals()['gradle'] = "gradle" common.read_config(config)
execfile('config.py', globals())
# Parse command line... # Parse command line...
parser = OptionParser() parser = OptionParser()
@ -334,13 +335,13 @@ def main():
tag = None tag = None
mode = app['Update Check Mode'] mode = app['Update Check Mode']
if mode == 'Tags': if mode == 'Tags':
(version, vercode, tag) = check_tags(app, sdk_path) (version, vercode, tag) = check_tags(app, config['sdk_path'])
elif mode == 'RepoManifest': elif mode == 'RepoManifest':
(version, vercode) = check_repomanifest(app, sdk_path) (version, vercode) = check_repomanifest(app, config['sdk_path'])
elif mode.startswith('RepoManifest/'): elif mode.startswith('RepoManifest/'):
(version, vercode) = check_repomanifest(app, sdk_path, mode[13:]) (version, vercode) = check_repomanifest(app, config['sdk_path'], mode[13:])
elif mode == 'RepoTrunk': elif mode == 'RepoTrunk':
(version, vercode) = check_repotrunk(app, sdk_path) (version, vercode) = check_repotrunk(app, config['sdk_path'])
elif mode == 'HTTP': elif mode == 'HTTP':
(version, vercode) = check_http(app) (version, vercode) = check_http(app)
elif mode == 'Static': elif mode == 'Static':
@ -374,7 +375,8 @@ def main():
else: else:
app_dir = os.path.join('build/', app['id']) app_dir = os.path.join('build/', app['id'])
vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir, sdk_path) vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir,
config['sdk_path'])
vcs.gotorevision(tag) vcs.gotorevision(tag)
flavour = None flavour = None

27
fdroidserver/common.py
View File

@ -27,6 +27,26 @@ import Queue
import threading import threading
import magic import magic
def read_config(config):
"""Read the repository config
The config is read from config.py, which is in the current directory when
any of the repo management commands are used.
"""
if not os.path.isfile('config.py'):
print "Missing config file - is this a repo directory?"
sys.exit(2)
config['build_server_always'] = False
config['mvn3'] = "mvn3"
config['archive_older'] = 0
config['gradle'] = 'gradle'
config['update_stats'] = False
config['archive_older'] = 0
config['max_icon_size'] = 72
config['stats_to_carbon'] = False
execfile('config.py', config)
def getvcs(vcstype, remote, local, sdk_path): def getvcs(vcstype, remote, local, sdk_path):
if vcstype == 'git': if vcstype == 'git':
return vcs_git(remote, local, sdk_path) return vcs_git(remote, local, sdk_path)
@ -1755,14 +1775,13 @@ class KnownApks:
lst.reverse() lst.reverse()
return lst return lst
def isApkDebuggable(apkfile): def isApkDebuggable(apkfile, config):
"""Returns True if the given apk file is debuggable """Returns True if the given apk file is debuggable
:param apkfile: full path to the apk to check""" :param apkfile: full path to the apk to check"""
execfile('config.py', globals()) p = subprocess.Popen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'),
p = subprocess.Popen([os.path.join(sdk_path, 'build-tools', build_tools, 'aapt'),
'dump', 'xmltree', apkfile, 'AndroidManifest.xml'], 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
stdout=subprocess.PIPE) stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]

9
fdroidserver/import.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# import.py - part of the FDroid server tools # import.py - part of the FDroid server tools
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by # it under the terms of the GNU Affero General Public License as published by
@ -22,7 +22,7 @@ import os
import shutil import shutil
import urllib import urllib
from optparse import OptionParser from optparse import OptionParser
import common
# Get the repo type and address from the given web page. The page is scanned # Get the repo type and address from the given web page. The page is scanned
# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and # in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
@ -82,11 +82,12 @@ def getrepofrompage(url):
return (None, "No information found." + page) return (None, "No information found." + page)
config = {}
def main(): def main():
# Read configuration... # Read configuration...
execfile('config.py', globals()) common.read_config(config)
import common import common
@ -219,7 +220,7 @@ def main():
src_dir = os.path.join(tmp_dir, 'importer') src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir): if os.path.exists(src_dir):
shutil.rmtree(src_dir) shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir, sdk_path) vcs = common.getvcs(repotype, repo, src_dir, config['sdk_path'])
vcs.gotorevision(options.rev) vcs.gotorevision(options.rev)
if options.subdir: if options.subdir:
root_dir = os.path.join(src_dir, options.subdir) root_dir = os.path.join(src_dir, options.subdir)

29
fdroidserver/publish.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# publish.py - part of the FDroid server tools # publish.py - part of the FDroid server tools
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc> # Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -26,12 +26,15 @@ import md5
import glob import glob
from optparse import OptionParser from optparse import OptionParser
import common
from common import BuildException from common import BuildException
config = {}
def main(): def main():
# Read configuration... # Read configuration...
execfile('config.py', globals()) common.read_config(config)
# Parse command line... # Parse command line...
parser = OptionParser() parser = OptionParser()
@ -78,9 +81,9 @@ def main():
# If a collision does occur later, we're going to have to # If a collision does occur later, we're going to have to
# come up with a new alogrithm, AND rename all existing keys # come up with a new alogrithm, AND rename all existing keys
# in the keystore! # in the keystore!
if appid in keyaliases: if appid in config['keyaliases']:
# For this particular app, the key alias is overridden... # For this particular app, the key alias is overridden...
keyalias = keyaliases[appid] keyalias = config['keyaliases'][appid]
if keyalias.startswith('@'): if keyalias.startswith('@'):
m = md5.new() m = md5.new()
m.update(keyalias[1:]) m.update(keyalias[1:])
@ -94,25 +97,27 @@ def main():
# See if we already have a key for this application, and # See if we already have a key for this application, and
# if not generate one... # if not generate one...
p = subprocess.Popen(['keytool', '-list', p = subprocess.Popen(['keytool', '-list',
'-alias', keyalias, '-keystore', keystore, '-alias', keyalias, '-keystore', config['keystore'],
'-storepass', keystorepass], stdout=subprocess.PIPE) '-storepass', config['keystorepass']], stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]
if p.returncode !=0: if p.returncode !=0:
print "Key does not exist - generating..." print "Key does not exist - generating..."
p = subprocess.Popen(['keytool', '-genkey', p = subprocess.Popen(['keytool', '-genkey',
'-keystore', keystore, '-alias', keyalias, '-keystore', config['keystore'], '-alias', keyalias,
'-keyalg', 'RSA', '-keysize', '2048', '-keyalg', 'RSA', '-keysize', '2048',
'-validity', '10000', '-validity', '10000',
'-storepass', keystorepass, '-keypass', keypass, '-storepass', config['keystorepass'],
'-dname', keydname], stdout=subprocess.PIPE) '-keypass', config['keypass'],
'-dname', config['keydname']], stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]
print output print output
if p.returncode != 0: if p.returncode != 0:
raise BuildException("Failed to generate key") raise BuildException("Failed to generate key")
# Sign the application... # Sign the application...
p = subprocess.Popen(['jarsigner', '-keystore', keystore, p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
'-storepass', keystorepass, '-keypass', keypass, '-sigalg', '-storepass', config['keystorepass'],
'-keypass', config['keypass'], '-sigalg',
'MD5withRSA', '-digestalg', 'SHA1', 'MD5withRSA', '-digestalg', 'SHA1',
apkfile, keyalias], stdout=subprocess.PIPE) apkfile, keyalias], stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]
@ -121,7 +126,7 @@ def main():
raise BuildException("Failed to sign application") raise BuildException("Failed to sign application")
# Zipalign it... # Zipalign it...
p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'), p = subprocess.Popen([os.path.join(config['sdk_path'],'tools','zipalign'),
'-v', '4', apkfile, '-v', '4', apkfile,
os.path.join(output_dir, apkfilename)], os.path.join(output_dir, apkfilename)],
stdout=subprocess.PIPE) stdout=subprocess.PIPE)

18
fdroidserver/scanner.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# scanner.py - part of the FDroid server tools # scanner.py - part of the FDroid server tools
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by # it under the terms of the GNU Affero General Public License as published by
@ -26,13 +26,12 @@ import common
from common import BuildException from common import BuildException
from common import VCSException from common import VCSException
config = {}
def main(): def main():
# Read configuration... # Read configuration...
global build_server_always, mvn3 common.read_config(config)
globals()['build_server_always'] = False
globals()['mvn3'] = "mvn3"
execfile('config.py', globals())
# Parse command line... # Parse command line...
@ -87,7 +86,8 @@ def main():
build_dir = 'build/' + app['id'] build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code... # Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir,
config['sdk_path'])
for thisbuild in app['builds']: for thisbuild in app['builds']:
@ -99,8 +99,10 @@ def main():
# Prepare the source code... # Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, thisbuild, root_dir, _ = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, build_dir, srclib_dir, extlib_dir,
javacc_path, mvn3, options.verbose, False) config['sdk_path'], config['ndk_path'],
config['javacc_path'], config['mvn3'],
options.verbose, False)
# Do the scan... # Do the scan...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild) buildprobs = common.scan_source(build_dir, root_dir, thisbuild)

17
fdroidserver/server.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# server.py - part of the FDroid server tools # server.py - part of the FDroid server tools
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by # it under the terms of the GNU Affero General Public License as published by
@ -21,13 +21,14 @@ import sys
import os import os
import subprocess import subprocess
from optparse import OptionParser from optparse import OptionParser
import common
config = {}
def main(): def main():
# Read configuration... # Read configuration...
global archive_older common.read_config(config)
archive_older = 0
execfile('config.py', globals())
# Parse command line... # Parse command line...
parser = OptionParser() parser = OptionParser()
@ -44,20 +45,20 @@ def main():
sys.exit(1) sys.exit(1)
repodirs = ['repo'] repodirs = ['repo']
if archive_older != 0: if config['archive_older'] != 0:
repodirs.append('archive') repodirs.append('archive')
for repodir in repodirs: for repodir in repodirs:
index = os.path.join(repodir, 'index.xml') index = os.path.join(repodir, 'index.xml')
indexjar = os.path.join(repodir, 'index.jar') indexjar = os.path.join(repodir, 'index.jar')
if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
'--exclude', index, '--exclude', indexjar, repodir, serverwebroot]) != 0: '--exclude', index, '--exclude', indexjar, repodir, config['serverwebroot']]) != 0:
sys.exit(1) sys.exit(1)
if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
index, serverwebroot + '/' + repodir]) != 0: index, config['serverwebroot'] + '/' + repodir]) != 0:
sys.exit(1) sys.exit(1)
if subprocess.call(['rsync', '-u', '-v', '-r', '--delete', if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
indexjar, serverwebroot + '/' + repodir]) != 0: indexjar, config['serverwebroot'] + '/' + repodir]) != 0:
sys.exit(1) sys.exit(1)
sys.exit(0) sys.exit(0)

17
fdroidserver/stats.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# stats.py - part of the FDroid server tools # stats.py - part of the FDroid server tools
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by # it under the terms of the GNU Affero General Public License as published by
@ -31,20 +31,19 @@ import subprocess
def carbon_send(key, value): def carbon_send(key, value):
s = socket.socket() s = socket.socket()
s.connect((carbon_host, carbon_port)) s.connect((config['carbon_host'], config['carbon_port']))
msg = '%s %d %d\n' % (key, value, int(time.time())) msg = '%s %d %d\n' % (key, value, int(time.time()))
s.sendall(msg) s.sendall(msg)
s.close() s.close()
config = {}
def main(): def main():
# Read configuration... # Read configuration...
global update_stats, stats_to_carbon common.read_config(config)
update_stats = False
stats_to_carbon = False
execfile('config.py', globals())
if not update_stats: if not config['update_stats']:
print "Stats are disabled - check your configuration" print "Stats are disabled - check your configuration"
sys.exit(1) sys.exit(1)
@ -79,7 +78,7 @@ def main():
ssh = paramiko.SSHClient() ssh = paramiko.SSHClient()
ssh.load_system_host_keys() ssh.load_system_host_keys()
ssh.connect('f-droid.org', username='fdroid', timeout=10, ssh.connect('f-droid.org', username='fdroid', timeout=10,
key_filename=webserver_keyfile) key_filename=config['webserver_keyfile'])
ftp = ssh.open_sftp() ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60) ftp.get_channel().settimeout(60)
print "...connected" print "...connected"
@ -140,7 +139,7 @@ def main():
alldownloads = 0 alldownloads = 0
for app, count in apps.iteritems(): for app, count in apps.iteritems():
lst.append(app + " " + str(count)) lst.append(app + " " + str(count))
if stats_to_carbon: if config['stats_to_carbon']:
carbon_send('fdroid.download.' + app.replace('.', '_'), count) carbon_send('fdroid.download.' + app.replace('.', '_'), count)
alldownloads += count alldownloads += count
lst.append("ALL " + str(alldownloads)) lst.append("ALL " + str(alldownloads))

67
fdroidserver/update.py
View File

@ -45,8 +45,9 @@ def update_wiki(apps, apks, verbose=False):
wikicat = 'Apps' wikicat = 'Apps'
wikiredircat = 'App Redirects' wikiredircat = 'App Redirects'
import mwclient import mwclient
site = mwclient.Site((wiki_protocol, wiki_server), path=wiki_path) site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
site.login(wiki_user, wiki_password) path=config['wiki_path'])
site.login(config['wiki_user'], config['wiki_password'])
generated_pages = {} generated_pages = {}
generated_redirects = {} generated_redirects = {}
for app in apps: for app in apps:
@ -247,9 +248,10 @@ def delete_disabled_builds(apps, apkcache, repodirs):
def resize_icon(iconpath): def resize_icon(iconpath):
try: try:
im = Image.open(iconpath) im = Image.open(iconpath)
if any(length > max_icon_size for length in im.size): if any(length > config['max_icon_size'] for length in im.size):
print iconpath, "is too large:", im.size print iconpath, "is too large:", im.size
im.thumbnail((max_icon_size, max_icon_size), Image.ANTIALIAS) im.thumbnail((config['max_icon_size'], config['max_icon_size']),
Image.ANTIALIAS)
print iconpath, "new size:", im.size print iconpath, "new size:", im.size
im.save(iconpath, "PNG") im.save(iconpath, "PNG")
else: else:
@ -321,7 +323,7 @@ def scan_apks(apps, apkcache, repodir, knownapks):
thisinfo['size'] = os.path.getsize(apkfile) thisinfo['size'] = os.path.getsize(apkfile)
thisinfo['permissions'] = [] thisinfo['permissions'] = []
thisinfo['features'] = [] thisinfo['features'] = []
p = subprocess.Popen([os.path.join(sdk_path, 'build-tools', build_tools, 'aapt'), p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
'dump', 'badging', apkfile], 'dump', 'badging', apkfile],
stdout=subprocess.PIPE) stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]
@ -369,7 +371,7 @@ def scan_apks(apps, apkcache, repodir, knownapks):
thisinfo['sdkversion'] = 0 thisinfo['sdkversion'] = 0
# Check for debuggable apks... # Check for debuggable apks...
if common.isApkDebuggable(apkfile): if common.isApkDebuggable(apkfile, config):
print "WARNING: {0} is debuggable... {1}".format(apkfile, line) print "WARNING: {0} is debuggable... {1}".format(apkfile, line)
# Calculate the sha256... # Calculate the sha256...
@ -450,17 +452,17 @@ def make_index(apps, apks, repodir, archive, categories):
repoel = doc.createElement("repo") repoel = doc.createElement("repo")
if archive: if archive:
repoel.setAttribute("name", archive_name) repoel.setAttribute("name", config['archive_name'])
repoel.setAttribute("icon", os.path.basename(archive_icon)) repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
repoel.setAttribute("url", archive_url) repoel.setAttribute("url", config['archive_url'])
addElement('description', archive_description, doc, repoel) addElement('description', config['archive_description'], doc, repoel)
else: else:
repoel.setAttribute("name", repo_name) repoel.setAttribute("name", config['repo_name'])
repoel.setAttribute("icon", os.path.basename(repo_icon)) repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
repoel.setAttribute("url", repo_url) repoel.setAttribute("url", config['repo_url'])
addElement('description', repo_description, doc, repoel) addElement('description', config['repo_description'], doc, repoel)
if repo_keyalias is not None: if config['repo_keyalias'] is not None:
# Generate a certificate fingerprint the same way keytool does it # Generate a certificate fingerprint the same way keytool does it
# (but with slightly different formatting) # (but with slightly different formatting)
@ -473,9 +475,9 @@ def make_index(apps, apks, repodir, archive, categories):
def extract_pubkey(): def extract_pubkey():
p = subprocess.Popen(['keytool', '-exportcert', p = subprocess.Popen(['keytool', '-exportcert',
'-alias', repo_keyalias, '-alias', config['repo_keyalias'],
'-keystore', keystore, '-keystore', config['keystore'],
'-storepass', keystorepass], '-storepass', config['keystorepass']],
stdout=subprocess.PIPE) stdout=subprocess.PIPE)
cert = p.communicate()[0] cert = p.communicate()[0]
if p.returncode != 0: if p.returncode != 0:
@ -606,11 +608,11 @@ def make_index(apps, apks, repodir, archive, categories):
of.write(output) of.write(output)
of.close() of.close()
if repo_keyalias is not None: if config['repo_keyalias'] is not None:
if not options.quiet: if not options.quiet:
print "Creating signed index." print "Creating signed index."
print "Key fingerprint:", repo_pubkey_fingerprint print "Key fingerprint:", config['repo_pubkey_fingerprint']
#Create a jar of the index... #Create a jar of the index...
p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'], p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
@ -623,10 +625,10 @@ def make_index(apps, apks, repodir, archive, categories):
sys.exit(1) sys.exit(1)
# Sign the index... # Sign the index...
p = subprocess.Popen(['jarsigner', '-keystore', keystore, p = subprocess.Popen(['jarsigner', '-keystore', config['keystore'],
'-storepass', keystorepass, '-keypass', keypass, '-storepass', config['keystorepass'], '-keypass', config['keypass'],
'-digestalg', 'SHA1', '-sigalg', 'MD5withRSA', '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
os.path.join(repodir, 'index.jar') , repo_keyalias], stdout=subprocess.PIPE) os.path.join(repodir, 'index.jar') , config['repo_keyalias']], stdout=subprocess.PIPE)
output = p.communicate()[0] output = p.communicate()[0]
if p.returncode != 0: if p.returncode != 0:
print "Failed to sign index" print "Failed to sign index"
@ -637,8 +639,8 @@ def make_index(apps, apks, repodir, archive, categories):
# Copy the repo icon into the repo directory... # Copy the repo icon into the repo directory...
icon_dir=os.path.join(repodir ,'icons') icon_dir=os.path.join(repodir ,'icons')
iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon)) iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon']))
shutil.copyfile(repo_icon, iconfilename) shutil.copyfile(config['repo_icon'], iconfilename)
# Write a category list in the repo to allow quick access... # Write a category list in the repo to allow quick access...
catdata = '' catdata = ''
@ -679,14 +681,13 @@ def archive_old_apks(apps, apks, repodir, archivedir, defaultkeepversions):
apks.remove(apk) apks.remove(apk)
config = {}
options = None
def main(): def main():
# Read configuration... # Read configuration...
global update_stats, archive_older, max_icon_size common.read_config(config)
update_stats = False
archive_older = 0
max_icon_size = 72
execfile('config.py', globals())
# Parse command line... # Parse command line...
global options global options
@ -715,7 +716,7 @@ def main():
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
repodirs = ['repo'] repodirs = ['repo']
if archive_older != 0: if config['archive_older'] != 0:
repodirs.append('archive') repodirs.append('archive')
if not os.path.exists('archive'): if not os.path.exists('archive'):
os.mkdir('archive') os.mkdir('archive')
@ -825,7 +826,7 @@ def main():
print " " + apk['name'] + " - " + apk['version'] print " " + apk['name'] + " - " + apk['version']
if len(repodirs) > 1: if len(repodirs) > 1:
archive_old_apks(apps, apks, repodirs[0], repodirs[1], archive_older) archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
# Make the index for the main repo... # Make the index for the main repo...
make_index(apps, apks, repodirs[0], False, categories) make_index(apps, apks, repodirs[0], False, categories)
@ -838,7 +839,7 @@ def main():
cachechanged = True cachechanged = True
make_index(apps, archapks, repodirs[1], True, categories) make_index(apps, archapks, repodirs[1], True, categories)
if update_stats: if config['update_stats']:
# Update known apks info... # Update known apks info...
knownapks.writeifchanged() knownapks.writeifchanged()