diff --git a/checkupdates.py b/checkupdates.py
index 2c083044..b679d6f8 100755
--- a/checkupdates.py
+++ b/checkupdates.py
@@ -27,9 +27,6 @@ from optparse import OptionParser
import HTMLParser
import common
-#Read configuration...
-execfile('config.py')
-
# Check for a new version by looking at the Google market.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
@@ -66,48 +63,55 @@ def check_market(app):
+def main():
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
-parser.add_option("-p", "--package", default=None,
- help="Build only the specified package")
-(options, args) = parser.parse_args()
+ #Read configuration...
+ execfile('config.py')
-# Get all apps...
-apps = common.read_metadata(options.verbose)
+ # Parse command line...
+ parser = OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_option("-p", "--package", default=None,
+ help="Build only the specified package")
+ (options, args) = parser.parse_args()
-html_parser = HTMLParser.HTMLParser()
+ # Get all apps...
+ apps = common.read_metadata(options.verbose)
-for app in apps:
+ html_parser = HTMLParser.HTMLParser()
- if options.package and options.package != app['id']:
- # Silent skip...
- pass
- else:
- print "Processing " + app['id'] + '...'
+ for app in apps:
- mode = app['Update Check Mode']
- if mode == 'Market':
- (version, vercode) = check_market(app)
- elif mode == 'None':
- version = None
- vercode = 'Checking disabled'
+ if options.package and options.package != app['id']:
+ # Silent skip...
+ pass
else:
- version = None
- vercode = 'Invalid update check method'
+ print "Processing " + app['id'] + '...'
- if not version:
- print "..." + vercode
- elif vercode == app['Current Version Code'] and version == app['Current Version']:
- print "...up to date"
- else:
- print '...updating to version:' + version + ' vercode:' + vercode
- app['Current Version'] = version
- app['Current Version Code'] = vercode
- metafile = os.path.join('metadata', app['id'] + '.txt')
- common.write_metadata(metafile, app)
+ mode = app['Update Check Mode']
+ if mode == 'Market':
+ (version, vercode) = check_market(app)
+ elif mode == 'None':
+ version = None
+ vercode = 'Checking disabled'
+ else:
+ version = None
+ vercode = 'Invalid update check method'
-print "Finished."
+ if not version:
+ print "..." + vercode
+ elif vercode == app['Current Version Code'] and version == app['Current Version']:
+ print "...up to date"
+ else:
+ print '...updating to version:' + version + ' vercode:' + vercode
+ app['Current Version'] = version
+ app['Current Version Code'] = vercode
+ metafile = os.path.join('metadata', app['id'] + '.txt')
+ common.write_metadata(metafile, app)
+
+ print "Finished."
+
+if __name__ == "__main__":
+ main()
diff --git a/import.py b/import.py
index e29fbc3a..3d794cb8 100755
--- a/import.py
+++ b/import.py
@@ -25,219 +25,225 @@ import re
import urllib
from optparse import OptionParser
-#Read configuration...
-repo_name = None
-repo_description = None
-repo_icon = None
-repo_url = None
-execfile('config.py')
+def main():
-import common
+ # Read configuration...
+ repo_name = None
+ repo_description = None
+ repo_icon = None
+ repo_url = None
+ execfile('config.py')
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-u", "--url", default=None,
- help="Project URL to import from.")
-parser.add_option("-s", "--subdir", default=None,
- help="Path to main android project subdirectory, if not in root.")
-(options, args) = parser.parse_args()
+ import common
-if not options.url:
- print "Specify project url."
- sys.exit(1)
-url = options.url
+ # Parse command line...
+ parser = OptionParser()
+ parser.add_option("-u", "--url", default=None,
+ help="Project URL to import from.")
+ parser.add_option("-s", "--subdir", default=None,
+ help="Path to main android project subdirectory, if not in root.")
+ (options, args) = parser.parse_args()
-tmp_dir = 'tmp'
-if not os.path.isdir(tmp_dir):
- print "Creating temporary directory"
- os.makedirs(tmp_dir)
-
-# Get all apps...
-apps = common.read_metadata()
-
-# Figure out what kind of project it is...
-projecttype = None
-issuetracker = None
-license = None
-if url.startswith('https://github.com'):
- projecttype = 'github'
- repo = url + '.git'
- repotype = 'git'
- sourcecode = url
-elif url.startswith('http://code.google.com/p/'):
- if not url.endswith('/'):
- print "Expected format for googlecode url is http://code.google.com/p/PROJECT/"
+ if not options.url:
+ print "Specify project url."
sys.exit(1)
- projecttype = 'googlecode'
- sourcecode = url + 'source/checkout'
- issuetracker = url + 'issues/list'
+ url = options.url
- # Figure out the repo type and adddress...
- req = urllib.urlopen(sourcecode)
- if req.getcode() != 200:
- print 'Unable to find source at ' + sourcecode + ' - return code ' + str(req.getcode())
- sys.exit(1)
- page = req.read()
- repotype = None
- index = page.find('hg clone')
- if index != -1:
- repotype = 'hg'
- repo = page[index + 9:]
- index = repo.find('<')
- if index == -1:
- print "Error while getting repo address"
+ tmp_dir = 'tmp'
+ if not os.path.isdir(tmp_dir):
+ print "Creating temporary directory"
+ os.makedirs(tmp_dir)
+
+ # Get all apps...
+ apps = common.read_metadata()
+
+ # Figure out what kind of project it is...
+ projecttype = None
+ issuetracker = None
+ license = None
+ if url.startswith('https://github.com'):
+ projecttype = 'github'
+ repo = url + '.git'
+ repotype = 'git'
+ sourcecode = url
+ elif url.startswith('http://code.google.com/p/'):
+ if not url.endswith('/'):
+ print "Expected format for googlecode url is http://code.google.com/p/PROJECT/"
sys.exit(1)
- repo = repo[:index]
- if not repotype:
- index=page.find('git clone')
+ projecttype = 'googlecode'
+ sourcecode = url + 'source/checkout'
+ issuetracker = url + 'issues/list'
+
+ # Figure out the repo type and adddress...
+ req = urllib.urlopen(sourcecode)
+ if req.getcode() != 200:
+ print 'Unable to find source at ' + sourcecode + ' - return code ' + str(req.getcode())
+ sys.exit(1)
+ page = req.read()
+ repotype = None
+ index = page.find('hg clone')
if index != -1:
- repotype = 'git'
- repo = page[index + 10:]
+ repotype = 'hg'
+ repo = page[index + 9:]
index = repo.find('<')
if index == -1:
print "Error while getting repo address"
sys.exit(1)
repo = repo[:index]
- if not repotype:
- index=page.find('svn checkout')
- if index != -1:
- repotype = 'git-svn'
- repo = page[index + 13:]
- prefix = 'http'
- if not repo.startswith(prefix):
- print "Unexpected checkout instructions format"
- sys.exit(1)
- repo = 'http' + repo[len(prefix):]
- index = repo.find('<')
- if index == -1:
- print "Error while getting repo address - no end tag? '" + repo + "'"
- sys.exit(1)
- repo = repo[:index]
- index = repo.find(' ')
- if index == -1:
- print "Error while getting repo address - no space? '" + repo + "'"
- sys.exit(1)
- repo = repo[:index]
- if not repotype:
- print "Unable to determine vcs type"
+ if not repotype:
+ index=page.find('git clone')
+ if index != -1:
+ repotype = 'git'
+ repo = page[index + 10:]
+ index = repo.find('<')
+ if index == -1:
+ print "Error while getting repo address"
+ sys.exit(1)
+ repo = repo[:index]
+ if not repotype:
+ index=page.find('svn checkout')
+ if index != -1:
+ repotype = 'git-svn'
+ repo = page[index + 13:]
+ prefix = 'http'
+ if not repo.startswith(prefix):
+ print "Unexpected checkout instructions format"
+ sys.exit(1)
+ repo = 'http' + repo[len(prefix):]
+ index = repo.find('<')
+ if index == -1:
+ print "Error while getting repo address - no end tag? '" + repo + "'"
+ sys.exit(1)
+ repo = repo[:index]
+ index = repo.find(' ')
+ if index == -1:
+ print "Error while getting repo address - no space? '" + repo + "'"
+ sys.exit(1)
+ repo = repo[:index]
+ if not repotype:
+ print "Unable to determine vcs type"
+ sys.exit(1)
+
+ # Figure out the license...
+ req = urllib.urlopen(url)
+ if req.getcode() != 200:
+ print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
+ sys.exit(1)
+ page = req.read()
+ index = page.find('Code license')
+ if index == -1:
+ print "Couldn't find license data"
+ sys.exit(1)
+ ltext = page[index:]
+ lprefix = 'rel="nofollow">'
+ index = ltext.find(lprefix)
+ if index == -1:
+ print "Couldn't find license text"
+ sys.exit(1)
+ ltext = ltext[index + len(lprefix):]
+ index = ltext.find('<')
+ if index == -1:
+ print "License text not formatted as expected"
+ sys.exit(1)
+ ltext = ltext[:index]
+ if ltext == 'GNU GPL v3':
+ license = 'GPLv3'
+ elif ltext == 'GNU GPL v2':
+ license = 'GPLv2'
+ elif ltext == 'Apache License 2.0':
+ license = 'Apache2'
+ else:
+ print "License " + ltext + " is not recognised"
+ sys.exit(1)
+
+ if not projecttype:
+ print "Unable to determine the project type."
sys.exit(1)
- # Figure out the license...
- req = urllib.urlopen(url)
- if req.getcode() != 200:
- print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
- sys.exit(1)
- page = req.read()
- index = page.find('Code license')
- if index == -1:
- print "Couldn't find license data"
- sys.exit(1)
- ltext = page[index:]
- lprefix = 'rel="nofollow">'
- index = ltext.find(lprefix)
- if index == -1:
- print "Couldn't find license text"
- sys.exit(1)
- ltext = ltext[index + len(lprefix):]
- index = ltext.find('<')
- if index == -1:
- print "License text not formatted as expected"
- sys.exit(1)
- ltext = ltext[:index]
- if ltext == 'GNU GPL v3':
- license = 'GPLv3'
- elif ltext == 'GNU GPL v2':
- license = 'GPLv2'
- elif ltext == 'Apache License 2.0':
- license = 'Apache2'
+ # Get a copy of the source so we can extract some info...
+ print 'Getting source from ' + repotype + ' repo at ' + repo
+ src_dir = os.path.join(tmp_dir, 'importer')
+ if os.path.exists(tmp_dir):
+ shutil.rmtree(tmp_dir)
+ vcs = common.getvcs(repotype, repo, src_dir)
+ vcs.gotorevision(None)
+ if options.subdir:
+ root_dir = os.path.join(src_dir, options.subdir)
else:
- print "License " + ltext + " is not recognised"
+ root_dir = src_dir
+
+ # Check AndroidManiifest.xml exists...
+ manifest = os.path.join(root_dir, 'AndroidManifest.xml')
+ if not os.path.exists(manifest):
+ print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
sys.exit(1)
-if not projecttype:
- print "Unable to determine the project type."
- sys.exit(1)
-
-# Get a copy of the source so we can extract some info...
-print 'Getting source from ' + repotype + ' repo at ' + repo
-src_dir = os.path.join(tmp_dir, 'importer')
-if os.path.exists(tmp_dir):
- shutil.rmtree(tmp_dir)
-vcs = common.getvcs(repotype, repo, src_dir)
-vcs.gotorevision(None)
-if options.subdir:
- root_dir = os.path.join(src_dir, options.subdir)
-else:
- root_dir = src_dir
-
-# Check AndroidManiifest.xml exists...
-manifest = os.path.join(root_dir, 'AndroidManifest.xml')
-if not os.path.exists(manifest):
- print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
- sys.exit(1)
-
-# Extract some information...
-vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search
-vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search
-psearch = re.compile(r'.*package="([^"]+)".*').search
-version = None
-vercode = None
-package = None
-for line in file(manifest):
+ # Extract some information...
+ vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search
+ vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search
+ psearch = re.compile(r'.*package="([^"]+)".*').search
+ version = None
+ vercode = None
+ package = None
+ for line in file(manifest):
+ if not package:
+ matches = psearch(line)
+ if matches:
+ package = matches.group(1)
+ if not version:
+ matches = vnsearch(line)
+ if matches:
+ version = matches.group(1)
+ if not vercode:
+ matches = vcsearch(line)
+ if matches:
+ vercode = matches.group(1)
if not package:
- matches = psearch(line)
- if matches:
- package = matches.group(1)
+ print "Couldn't find package ID"
+ sys.exit(1)
if not version:
- matches = vnsearch(line)
- if matches:
- version = matches.group(1)
+ print "Couldn't find latest version name"
+ sys.exit(1)
if not vercode:
- matches = vcsearch(line)
- if matches:
- vercode = matches.group(1)
-if not package:
- print "Couldn't find package ID"
- sys.exit(1)
-if not version:
- print "Couldn't find latest version name"
- sys.exit(1)
-if not vercode:
- print "Couldn't find latest version code"
- sys.exit(1)
-
-# Make sure it's actually new...
-for app in apps:
- if app['id'] == package:
- print "Package " + package + " already exists"
+ print "Couldn't find latest version code"
sys.exit(1)
-# Construct the metadata...
-app = common.parse_metadata(None)
-app['id'] = package
-app['Web Site'] = url
-app['Source Code'] = sourcecode
-if issuetracker:
- app['Issue Tracker'] = issuetracker
-if license:
- app['License'] = license
-app['Repo Type'] = repotype
-app['Repo'] = repo
+ # Make sure it's actually new...
+ for app in apps:
+ if app['id'] == package:
+ print "Package " + package + " already exists"
+ sys.exit(1)
-# Create a build line...
-build = {}
-build['version'] = version
-build['vercode'] = vercode
-build['commit'] = '?'
-if options.subdir:
- build['subdir'] = options.subdir
-if os.path.exists(os.path.join(root_dir, 'jni')):
- build['buildjni'] = 'yes'
-app['builds'].append(build)
-app['comments'].append(('build:' + version,
- "#Generated by import.py - check this is the right version, and find the right commit!"))
+ # Construct the metadata...
+ app = common.parse_metadata(None)
+ app['id'] = package
+ app['Web Site'] = url
+ app['Source Code'] = sourcecode
+ if issuetracker:
+ app['Issue Tracker'] = issuetracker
+ if license:
+ app['License'] = license
+ app['Repo Type'] = repotype
+ app['Repo'] = repo
-metafile = os.path.join('metadata', package + '.txt')
-common.write_metadata(metafile, app)
-print "Wrote " + metafile
+ # Create a build line...
+ build = {}
+ build['version'] = version
+ build['vercode'] = vercode
+ build['commit'] = '?'
+ if options.subdir:
+ build['subdir'] = options.subdir
+ if os.path.exists(os.path.join(root_dir, 'jni')):
+ build['buildjni'] = 'yes'
+ app['builds'].append(build)
+ app['comments'].append(('build:' + version,
+ "#Generated by import.py - check this is the right version, and find the right commit!"))
+
+ metafile = os.path.join('metadata', package + '.txt')
+ common.write_metadata(metafile, app)
+ print "Wrote " + metafile
+
+
+if __name__ == "__main__":
+ main()
diff --git a/publish.py b/publish.py
index 7fd9a1fe..c5610ffe 100755
--- a/publish.py
+++ b/publish.py
@@ -31,106 +31,112 @@ from optparse import OptionParser
import common
from common import BuildException
-#Read configuration...
-execfile('config.py')
+def main():
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
-parser.add_option("-p", "--package", default=None,
- help="Publish only the specified package")
-(options, args) = parser.parse_args()
+ #Read configuration...
+ execfile('config.py')
-log_dir = 'logs'
-if not os.path.isdir(log_dir):
- print "Creating log directory"
- os.makedirs(log_dir)
+ # Parse command line...
+ parser = OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_option("-p", "--package", default=None,
+ help="Publish only the specified package")
+ (options, args) = parser.parse_args()
-tmp_dir = 'tmp'
-if not os.path.isdir(tmp_dir):
- print "Creating temporary directory"
- os.makedirs(tmp_dir)
+ log_dir = 'logs'
+ if not os.path.isdir(log_dir):
+ print "Creating log directory"
+ os.makedirs(log_dir)
-output_dir = 'repo'
-if not os.path.isdir(output_dir):
- print "Creating output directory"
- os.makedirs(output_dir)
+ tmp_dir = 'tmp'
+ if not os.path.isdir(tmp_dir):
+ print "Creating temporary directory"
+ os.makedirs(tmp_dir)
-unsigned_dir = 'unsigned'
-if not os.path.isdir(unsigned_dir):
- print "No unsigned directory - nothing to do"
- sys.exit(0)
+ output_dir = 'repo'
+ if not os.path.isdir(output_dir):
+ print "Creating output directory"
+ os.makedirs(output_dir)
-for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
+ unsigned_dir = 'unsigned'
+ if not os.path.isdir(unsigned_dir):
+ print "No unsigned directory - nothing to do"
+ sys.exit(0)
- apkfilename = os.path.basename(apkfile)
- i = apkfilename.rfind('_')
- if i == -1:
- raise BuildException("Invalid apk name")
- appid = apkfilename[:i]
- print "Processing " + appid
+ for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
- if not options.package or options.package == appid:
+ apkfilename = os.path.basename(apkfile)
+ i = apkfilename.rfind('_')
+ if i == -1:
+ raise BuildException("Invalid apk name")
+ appid = apkfilename[:i]
+ print "Processing " + appid
- # Figure out the key alias name we'll use. Only the first 8
- # characters are significant, so we'll use the first 8 from
- # the MD5 of the app's ID and hope there are no collisions.
- # If a collision does occur later, we're going to have to
- # come up with a new alogrithm, AND rename all existing keys
- # in the keystore!
- if keyaliases.has_key(appid):
- # For this particular app, the key alias is overridden...
- keyalias = keyaliases[appid]
- else:
- m = md5.new()
- m.update(appid)
- keyalias = m.hexdigest()[:8]
- print "Key alias: " + keyalias
+ if not options.package or options.package == appid:
- # See if we already have a key for this application, and
- # if not generate one...
- p = subprocess.Popen(['keytool', '-list',
- '-alias', keyalias, '-keystore', keystore,
- '-storepass', keystorepass], stdout=subprocess.PIPE)
- output = p.communicate()[0]
- if p.returncode !=0:
- print "Key does not exist - generating..."
- p = subprocess.Popen(['keytool', '-genkey',
- '-keystore', keystore, '-alias', keyalias,
- '-keyalg', 'RSA', '-keysize', '2048',
- '-validity', '10000',
+ # Figure out the key alias name we'll use. Only the first 8
+ # characters are significant, so we'll use the first 8 from
+ # the MD5 of the app's ID and hope there are no collisions.
+ # If a collision does occur later, we're going to have to
+ # come up with a new alogrithm, AND rename all existing keys
+ # in the keystore!
+ if keyaliases.has_key(appid):
+ # For this particular app, the key alias is overridden...
+ keyalias = keyaliases[appid]
+ else:
+ m = md5.new()
+ m.update(appid)
+ keyalias = m.hexdigest()[:8]
+ print "Key alias: " + keyalias
+
+ # See if we already have a key for this application, and
+ # if not generate one...
+ p = subprocess.Popen(['keytool', '-list',
+ '-alias', keyalias, '-keystore', keystore,
+ '-storepass', keystorepass], stdout=subprocess.PIPE)
+ output = p.communicate()[0]
+ if p.returncode !=0:
+ print "Key does not exist - generating..."
+ p = subprocess.Popen(['keytool', '-genkey',
+ '-keystore', keystore, '-alias', keyalias,
+ '-keyalg', 'RSA', '-keysize', '2048',
+ '-validity', '10000',
+ '-storepass', keystorepass, '-keypass', keypass,
+ '-dname', keydname], stdout=subprocess.PIPE)
+ output = p.communicate()[0]
+ print output
+ if p.returncode != 0:
+ raise BuildException("Failed to generate key")
+
+ # Sign the application...
+ p = subprocess.Popen(['jarsigner', '-keystore', keystore,
'-storepass', keystorepass, '-keypass', keypass,
- '-dname', keydname], stdout=subprocess.PIPE)
+ apkfile, keyalias], stdout=subprocess.PIPE)
output = p.communicate()[0]
print output
if p.returncode != 0:
- raise BuildException("Failed to generate key")
+ raise BuildException("Failed to sign application")
- # Sign the application...
- p = subprocess.Popen(['jarsigner', '-keystore', keystore,
- '-storepass', keystorepass, '-keypass', keypass,
- apkfile, keyalias], stdout=subprocess.PIPE)
- output = p.communicate()[0]
- print output
- if p.returncode != 0:
- raise BuildException("Failed to sign application")
+ # Zipalign it...
+ p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'),
+ '-v', '4', apkfile,
+ os.path.join(output_dir, apkfilename)],
+ stdout=subprocess.PIPE)
+ output = p.communicate()[0]
+ print output
+ if p.returncode != 0:
+ raise BuildException("Failed to align application")
+ os.remove(apkfile)
- # Zipalign it...
- p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'),
- '-v', '4', apkfile,
- os.path.join(output_dir, apkfilename)],
- stdout=subprocess.PIPE)
- output = p.communicate()[0]
- print output
- if p.returncode != 0:
- raise BuildException("Failed to align application")
- os.remove(apkfile)
+ # Move the source tarball into the output directory...
+ tarfilename = apkfilename[:-4] + '_src.tar.gz'
+ shutil.move(os.path.join(unsigned_dir, tarfilename),
+ os.path.join(output_dir, tarfilename))
- # Move the source tarball into the output directory...
- tarfilename = apkfilename[:-4] + '_src.tar.gz'
- shutil.move(os.path.join(unsigned_dir, tarfilename),
- os.path.join(output_dir, tarfilename))
+ print 'Published ' + apkfilename
- print 'Published ' + apkfilename
+
+if __name__ == "__main__":
+ main()
diff --git a/rewritemeta.py b/rewritemeta.py
index e3535da2..b633f0cc 100755
--- a/rewritemeta.py
+++ b/rewritemeta.py
@@ -27,22 +27,26 @@ from optparse import OptionParser
import HTMLParser
import common
-#Read configuration...
-execfile('config.py')
+def main():
+ #Read configuration...
+ execfile('config.py')
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
-(options, args) = parser.parse_args()
+ # Parse command line...
+ parser = OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ (options, args) = parser.parse_args()
-# Get all apps...
-apps = common.read_metadata(options.verbose)
+ # Get all apps...
+ apps = common.read_metadata(options.verbose)
-for app in apps:
- print "Writing " + app['id']
- common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
+ for app in apps:
+ print "Writing " + app['id']
+ common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
-print "Finished."
+ print "Finished."
+
+if __name__ == "__main__":
+ main()
diff --git a/scanner.py b/scanner.py
index ee00a3ca..ad4e20fb 100755
--- a/scanner.py
+++ b/scanner.py
@@ -31,80 +31,85 @@ import common
from common import BuildException
from common import VCSException
-#Read configuration...
-execfile('config.py')
+def main():
+
+ # Read configuration...
+ execfile('config.py')
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
-parser.add_option("-p", "--package", default=None,
- help="Scan only the specified package")
-(options, args) = parser.parse_args()
+ # Parse command line...
+ parser = OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_option("-p", "--package", default=None,
+ help="Scan only the specified package")
+ (options, args) = parser.parse_args()
-# Get all apps...
-apps = common.read_metadata(options.verbose)
+ # Get all apps...
+ apps = common.read_metadata(options.verbose)
-html_parser = HTMLParser.HTMLParser()
+ html_parser = HTMLParser.HTMLParser()
-problems = []
+ problems = []
-extlib_dir = os.path.join('build', 'extlib')
+ extlib_dir = os.path.join('build', 'extlib')
-for app in apps:
+ for app in apps:
- skip = False
- if options.package and app['id'] != options.package:
- skip = True
- elif app['Disabled']:
- print "Skipping %s: disabled" % app['id']
- skip = True
- elif not app['builds']:
- print "Skipping %s: no builds specified" % app['id']
- skip = True
+ skip = False
+ if options.package and app['id'] != options.package:
+ skip = True
+ elif app['Disabled']:
+ print "Skipping %s: disabled" % app['id']
+ skip = True
+ elif not app['builds']:
+ print "Skipping %s: no builds specified" % app['id']
+ skip = True
- if not skip:
+ if not skip:
- print "Processing " + app['id']
+ print "Processing " + app['id']
- try:
+ try:
- build_dir = 'build/' + app['id']
+ build_dir = 'build/' + app['id']
- # Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+ # Set up vcs interface and make sure we have the latest code...
+ vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
- for thisbuild in app['builds']:
+ for thisbuild in app['builds']:
- if thisbuild['commit'].startswith('!'):
- print ("..skipping version " + thisbuild['version'] + " - " +
- thisbuild['commit'][1:])
- else:
- print "..scanning version " + thisbuild['version']
+ if thisbuild['commit'].startswith('!'):
+ print ("..skipping version " + thisbuild['version'] + " - " +
+ thisbuild['commit'][1:])
+ else:
+ print "..scanning version " + thisbuild['version']
- # Prepare the source code...
- root_dir = common.prepare_source(vcs, app, thisbuild,
- build_dir, extlib_dir, sdk_path, ndk_path, javacc_path)
+ # Prepare the source code...
+ root_dir = common.prepare_source(vcs, app, thisbuild,
+ build_dir, extlib_dir, sdk_path, ndk_path, javacc_path)
- # Do the scan...
- buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
- for problem in buildprobs:
- problems.append(problem +
- ' in ' + app['id'] + ' ' + thisbuild['version'])
+ # Do the scan...
+ buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
+ for problem in buildprobs:
+ problems.append(problem +
+ ' in ' + app['id'] + ' ' + thisbuild['version'])
- except BuildException as be:
- msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
- problems.append(msg)
- except VCSException as vcse:
- msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
- problems.append(msg)
- except Exception:
- msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
- problems.append(msg)
+ except BuildException as be:
+ msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
+ problems.append(msg)
+ except VCSException as vcse:
+ msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
+ problems.append(msg)
+ except Exception:
+ msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+ problems.append(msg)
-print "Finished:"
-for problem in problems:
- print problem
-print str(len(problems)) + ' problems.'
+ print "Finished:"
+ for problem in problems:
+ print problem
+ print str(len(problems)) + ' problems.'
+
+if __name__ == "__main__":
+ main()
diff --git a/update.py b/update.py
index ad0f2377..f378db7c 100755
--- a/update.py
+++ b/update.py
@@ -29,491 +29,496 @@ from xml.dom.minidom import Document
from optparse import OptionParser
import time
-#Read configuration...
-repo_name = None
-repo_description = None
-repo_icon = None
-repo_url = None
-execfile('config.py')
+def main():
-import common
+ # Read configuration...
+ repo_name = None
+ repo_description = None
+ repo_icon = None
+ repo_url = None
+ execfile('config.py')
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-c", "--createmeta", action="store_true", default=False,
- help="Create skeleton metadata files that are missing")
-parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
-parser.add_option("-q", "--quiet", action="store_true", default=False,
- help="No output, except for warnings and errors")
-parser.add_option("-b", "--buildreport", action="store_true", default=False,
- help="Report on build data status")
-parser.add_option("-i", "--interactive", default=False, action="store_true",
- help="Interactively ask about things that need updating.")
-parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
- help="Specify editor to use in interactive mode. Default "+
- "is /etc/alternatives/editor")
-parser.add_option("", "--pretty", action="store_true", default=False,
- help="Produce human-readable index.xml")
-(options, args) = parser.parse_args()
+ import common
+
+ # Parse command line...
+ parser = OptionParser()
+ parser.add_option("-c", "--createmeta", action="store_true", default=False,
+ help="Create skeleton metadata files that are missing")
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_option("-q", "--quiet", action="store_true", default=False,
+ help="No output, except for warnings and errors")
+ parser.add_option("-b", "--buildreport", action="store_true", default=False,
+ help="Report on build data status")
+ parser.add_option("-i", "--interactive", default=False, action="store_true",
+ help="Interactively ask about things that need updating.")
+ parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
+ help="Specify editor to use in interactive mode. Default "+
+ "is /etc/alternatives/editor")
+ parser.add_option("", "--pretty", action="store_true", default=False,
+ help="Produce human-readable index.xml")
+ (options, args) = parser.parse_args()
-icon_dir=os.path.join('repo','icons')
+ icon_dir=os.path.join('repo','icons')
-# Delete and re-create the icon directory...
-if os.path.exists(icon_dir):
- shutil.rmtree(icon_dir)
-os.mkdir(icon_dir)
+ # Delete and re-create the icon directory...
+ if os.path.exists(icon_dir):
+ shutil.rmtree(icon_dir)
+ os.mkdir(icon_dir)
-warnings = 0
+ warnings = 0
-#Make sure we have the repository description...
-if (repo_url is None or repo_name is None or
- repo_icon is None or repo_description is None):
- print "Repository description fields are required in config.py"
- print "See config.sample.py for details"
- sys.exit(1)
-
-# Get all apps...
-apps = common.read_metadata(verbose=options.verbose)
-
-# Generate a list of categories...
-categories = []
-for app in apps:
- if app['Category'] not in categories:
- categories.append(app['Category'])
-
-# Gather information about all the apk files in the repo directory...
-apks = []
-for apkfile in glob.glob(os.path.join('repo','*.apk')):
-
- apkfilename = apkfile[5:]
- if apkfilename.find(' ') != -1:
- print "No spaces in APK filenames!"
+ # Make sure we have the repository description...
+ if (repo_url is None or repo_name is None or
+ repo_icon is None or repo_description is None):
+ print "Repository description fields are required in config.py"
+ print "See config.sample.py for details"
sys.exit(1)
- srcfilename = apkfilename[:-4] + "_src.tar.gz"
- if not options.quiet:
- print "Processing " + apkfilename
- thisinfo = {}
- thisinfo['apkname'] = apkfilename
- if os.path.exists(os.path.join('repo', srcfilename)):
- thisinfo['srcname'] = srcfilename
- thisinfo['size'] = os.path.getsize(apkfile)
- thisinfo['permissions'] = []
- thisinfo['features'] = []
- p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
- 'dump', 'badging', apkfile],
- stdout=subprocess.PIPE)
- output = p.communicate()[0]
- if options.verbose:
- print output
- if p.returncode != 0:
- print "ERROR: Failed to get apk information"
- sys.exit(1)
- for line in output.splitlines():
- if line.startswith("package:"):
- pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
- thisinfo['id'] = re.match(pat, line).group(1)
- pat = re.compile(".*versionCode='([0-9]*)'.*")
- thisinfo['versioncode'] = int(re.match(pat, line).group(1))
- pat = re.compile(".*versionName='([^']*)'.*")
- thisinfo['version'] = re.match(pat, line).group(1)
- if line.startswith("application:"):
- pat = re.compile(".*label='([^']*)'.*")
- thisinfo['name'] = re.match(pat, line).group(1)
- pat = re.compile(".*icon='([^']*)'.*")
- thisinfo['iconsrc'] = re.match(pat, line).group(1)
- if line.startswith("sdkVersion:"):
- pat = re.compile(".*'([0-9]*)'.*")
- thisinfo['sdkversion'] = re.match(pat, line).group(1)
- if line.startswith("native-code:"):
- pat = re.compile(".*'([^']*)'.*")
- thisinfo['nativecode'] = re.match(pat, line).group(1)
- if line.startswith("uses-permission:"):
- pat = re.compile(".*'([^']*)'.*")
- perm = re.match(pat, line).group(1)
- if perm.startswith("android.permission."):
- perm = perm[19:]
- thisinfo['permissions'].append(perm)
- if line.startswith("uses-feature:"):
- pat = re.compile(".*'([^']*)'.*")
- perm = re.match(pat, line).group(1)
- #Filter out this, it's only added with the latest SDK tools and
- #causes problems for lots of apps.
- if (perm != "android.hardware.screen.portrait" and
- perm != "android.hardware.screen.landscape"):
- if perm.startswith("android.feature."):
- perm = perm[16:]
- thisinfo['features'].append(perm)
+ # Get all apps...
+ apps = common.read_metadata(verbose=options.verbose)
- if not thisinfo.has_key('sdkversion'):
- print " WARNING: no SDK version information found"
- thisinfo['sdkversion'] = 0
-
- # Calculate the md5 and sha256...
- m = hashlib.md5()
- sha = hashlib.sha256()
- f = open(apkfile, 'rb')
- while True:
- t = f.read(1024)
- if len(t) == 0:
- break
- m.update(t)
- sha.update(t)
- thisinfo['md5'] = m.hexdigest()
- thisinfo['sha256'] = sha.hexdigest()
- f.close()
-
- # Get the signature (or md5 of, to be precise)...
- p = subprocess.Popen(['java', 'getsig',
- os.path.join(os.getcwd(), apkfile)],
- cwd=os.path.join(sys.path[0], 'getsig'),
- stdout=subprocess.PIPE)
- output = p.communicate()[0]
- if options.verbose:
- print output
- if p.returncode != 0 or not output.startswith('Result:'):
- print "ERROR: Failed to get apk signature"
- sys.exit(1)
- thisinfo['sig'] = output[7:].strip()
-
- # Extract the icon file...
- apk = zipfile.ZipFile(apkfile, 'r')
- thisinfo['icon'] = (thisinfo['id'] + '.' +
- str(thisinfo['versioncode']) + '.png')
- iconfilename = os.path.join(icon_dir, thisinfo['icon'])
- try:
- iconfile = open(iconfilename, 'wb')
- iconfile.write(apk.read(thisinfo['iconsrc']))
- iconfile.close()
- except:
- print "WARNING: Error retrieving icon file"
- warnings += 1
- apk.close()
-
- apks.append(thisinfo)
-
-# Some information from the apks needs to be applied up to the application
-# level. When doing this, we use the info from the most recent version's apk.
-for app in apps:
- bestver = 0
- for apk in apks:
- if apk['id'] == app['id']:
- if apk['versioncode'] > bestver:
- bestver = apk['versioncode']
- bestapk = apk
-
- if bestver == 0:
- if app['Name'] is None:
- app['Name'] = app['id']
- app['icon'] = ''
- if app['Disabled'] is None:
- print "WARNING: Application " + app['id'] + " has no packages"
- else:
- if app['Name'] is None:
- app['Name'] = bestapk['name']
- app['icon'] = bestapk['icon']
-
-# Generate warnings for apk's with no metadata (or create skeleton
-# metadata files, if requested on the command line)
-for apk in apks:
- found = False
+ # Generate a list of categories...
+ categories = []
for app in apps:
- if app['id'] == apk['id']:
- found = True
- break
- if not found:
- if options.createmeta:
- f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
- f.write("License:Unknown\n")
- f.write("Web Site:\n")
- f.write("Source Code:\n")
- f.write("Issue Tracker:\n")
- f.write("Summary:" + apk['name'] + "\n")
- f.write("Description:\n")
- f.write(apk['name'] + "\n")
- f.write(".\n")
- f.close()
- print "Generated skeleton metadata for " + apk['id']
- else:
- print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
- print " " + apk['name'] + " - " + apk['version']
+ if app['Category'] not in categories:
+ categories.append(app['Category'])
-#Sort the app list by name, then the web site doesn't have to by default:
-apps = sorted(apps, key=lambda app: app['Name'].upper())
+ # Gather information about all the apk files in the repo directory...
+ apks = []
+ for apkfile in glob.glob(os.path.join('repo','*.apk')):
-# Create the index
-doc = Document()
-
-def addElement(name, value, doc, parent):
- el = doc.createElement(name)
- el.appendChild(doc.createTextNode(value))
- parent.appendChild(el)
-
-root = doc.createElement("fdroid")
-doc.appendChild(root)
-
-repoel = doc.createElement("repo")
-repoel.setAttribute("name", repo_name)
-repoel.setAttribute("icon", os.path.basename(repo_icon))
-repoel.setAttribute("url", repo_url)
-
-if repo_keyalias != None:
-
- # Generate a certificate fingerprint the same way keytool does it
- # (but with slightly different formatting)
- def cert_fingerprint(data):
- digest = hashlib.sha1(data).digest()
- ret = []
- for i in range(4):
- ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
- return " ".join(ret)
-
- def extract_pubkey():
- p = subprocess.Popen(['keytool', '-exportcert',
- '-alias', repo_keyalias,
- '-keystore', keystore,
- '-storepass', keystorepass],
- stdout=subprocess.PIPE)
- cert = p.communicate()[0]
- if p.returncode != 0:
- print "ERROR: Failed to get repo pubkey"
+ apkfilename = apkfile[5:]
+ if apkfilename.find(' ') != -1:
+ print "No spaces in APK filenames!"
sys.exit(1)
- global repo_pubkey_fingerprint
- repo_pubkey_fingerprint = cert_fingerprint(cert)
- return "".join("%02x" % ord(b) for b in cert)
+ srcfilename = apkfilename[:-4] + "_src.tar.gz"
- repoel.setAttribute("pubkey", extract_pubkey())
+ if not options.quiet:
+ print "Processing " + apkfilename
+ thisinfo = {}
+ thisinfo['apkname'] = apkfilename
+ if os.path.exists(os.path.join('repo', srcfilename)):
+ thisinfo['srcname'] = srcfilename
+ thisinfo['size'] = os.path.getsize(apkfile)
+ thisinfo['permissions'] = []
+ thisinfo['features'] = []
+ p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
+ 'dump', 'badging', apkfile],
+ stdout=subprocess.PIPE)
+ output = p.communicate()[0]
+ if options.verbose:
+ print output
+ if p.returncode != 0:
+ print "ERROR: Failed to get apk information"
+ sys.exit(1)
+ for line in output.splitlines():
+ if line.startswith("package:"):
+ pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
+ thisinfo['id'] = re.match(pat, line).group(1)
+ pat = re.compile(".*versionCode='([0-9]*)'.*")
+ thisinfo['versioncode'] = int(re.match(pat, line).group(1))
+ pat = re.compile(".*versionName='([^']*)'.*")
+ thisinfo['version'] = re.match(pat, line).group(1)
+ if line.startswith("application:"):
+ pat = re.compile(".*label='([^']*)'.*")
+ thisinfo['name'] = re.match(pat, line).group(1)
+ pat = re.compile(".*icon='([^']*)'.*")
+ thisinfo['iconsrc'] = re.match(pat, line).group(1)
+ if line.startswith("sdkVersion:"):
+ pat = re.compile(".*'([0-9]*)'.*")
+ thisinfo['sdkversion'] = re.match(pat, line).group(1)
+ if line.startswith("native-code:"):
+ pat = re.compile(".*'([^']*)'.*")
+ thisinfo['nativecode'] = re.match(pat, line).group(1)
+ if line.startswith("uses-permission:"):
+ pat = re.compile(".*'([^']*)'.*")
+ perm = re.match(pat, line).group(1)
+ if perm.startswith("android.permission."):
+ perm = perm[19:]
+ thisinfo['permissions'].append(perm)
+ if line.startswith("uses-feature:"):
+ pat = re.compile(".*'([^']*)'.*")
+ perm = re.match(pat, line).group(1)
+ #Filter out this, it's only added with the latest SDK tools and
+ #causes problems for lots of apps.
+ if (perm != "android.hardware.screen.portrait" and
+ perm != "android.hardware.screen.landscape"):
+ if perm.startswith("android.feature."):
+ perm = perm[16:]
+ thisinfo['features'].append(perm)
-addElement('description', repo_description, doc, repoel)
-root.appendChild(repoel)
+ if not thisinfo.has_key('sdkversion'):
+ print " WARNING: no SDK version information found"
+ thisinfo['sdkversion'] = 0
-apps_inrepo = 0
-apps_disabled = 0
-apps_nopkg = 0
+ # Calculate the md5 and sha256...
+ m = hashlib.md5()
+ sha = hashlib.sha256()
+ f = open(apkfile, 'rb')
+ while True:
+ t = f.read(1024)
+ if len(t) == 0:
+ break
+ m.update(t)
+ sha.update(t)
+ thisinfo['md5'] = m.hexdigest()
+ thisinfo['sha256'] = sha.hexdigest()
+ f.close()
-for app in apps:
+ # Get the signature (or md5 of, to be precise)...
+ p = subprocess.Popen(['java', 'getsig',
+ os.path.join(os.getcwd(), apkfile)],
+ cwd=os.path.join(sys.path[0], 'getsig'),
+ stdout=subprocess.PIPE)
+ output = p.communicate()[0]
+ if options.verbose:
+ print output
+ if p.returncode != 0 or not output.startswith('Result:'):
+ print "ERROR: Failed to get apk signature"
+ sys.exit(1)
+ thisinfo['sig'] = output[7:].strip()
- if app['Disabled'] is None:
+ # Extract the icon file...
+ apk = zipfile.ZipFile(apkfile, 'r')
+ thisinfo['icon'] = (thisinfo['id'] + '.' +
+ str(thisinfo['versioncode']) + '.png')
+ iconfilename = os.path.join(icon_dir, thisinfo['icon'])
+ try:
+ iconfile = open(iconfilename, 'wb')
+ iconfile.write(apk.read(thisinfo['iconsrc']))
+ iconfile.close()
+ except:
+ print "WARNING: Error retrieving icon file"
+ warnings += 1
+ apk.close()
- # Get a list of the apks for this app...
- gotcurrentver = False
- apklist = []
+ apks.append(thisinfo)
+
+ # Some information from the apks needs to be applied up to the application
+ # level. When doing this, we use the info from the most recent version's apk.
+ for app in apps:
+ bestver = 0
for apk in apks:
if apk['id'] == app['id']:
- if str(apk['versioncode']) == app['Current Version Code']:
- gotcurrentver = True
- apklist.append(apk)
+ if apk['versioncode'] > bestver:
+ bestver = apk['versioncode']
+ bestapk = apk
- if len(apklist) == 0:
- apps_nopkg += 1
+ if bestver == 0:
+ if app['Name'] is None:
+ app['Name'] = app['id']
+ app['icon'] = ''
+ if app['Disabled'] is None:
+ print "WARNING: Application " + app['id'] + " has no packages"
else:
- apps_inrepo += 1
- apel = doc.createElement("application")
- apel.setAttribute("id", app['id'])
- root.appendChild(apel)
+ if app['Name'] is None:
+ app['Name'] = bestapk['name']
+ app['icon'] = bestapk['icon']
- addElement('id', app['id'], doc, apel)
- addElement('name', app['Name'], doc, apel)
- addElement('summary', app['Summary'], doc, apel)
- addElement('icon', app['icon'], doc, apel)
- addElement('description',
- common.parse_description(app['Description']), doc, apel)
- addElement('license', app['License'], doc, apel)
- if 'Category' in app:
- addElement('category', app['Category'], doc, apel)
- addElement('web', app['Web Site'], doc, apel)
- addElement('source', app['Source Code'], doc, apel)
- addElement('tracker', app['Issue Tracker'], doc, apel)
- if app['Donate'] != None:
- addElement('donate', app['Donate'], doc, apel)
-
- # These elements actually refer to the current version (i.e. which
- # one is recommended. They are historically mis-named, and need
- # changing, but stay like this for now to support existing clients.
- addElement('marketversion', app['Current Version'], doc, apel)
- addElement('marketvercode', app['Current Version Code'], doc, apel)
-
- if not (app['AntiFeatures'] is None):
- addElement('antifeatures', app['AntiFeatures'], doc, apel)
- if app['Requires Root']:
- addElement('requirements', 'root', doc, apel)
-
- # Sort the apk list into version order, just so the web site
- # doesn't have to do any work by default...
- apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
-
- # Check for duplicates - they will make the client unhappy...
- for i in range(len(apklist) - 1):
- if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
- print "ERROR - duplicate versions"
- print apklist[i]['apkname']
- print apklist[i+1]['apkname']
- sys.exit(1)
-
- for apk in apklist:
- apkel = doc.createElement("package")
- apel.appendChild(apkel)
- addElement('version', apk['version'], doc, apkel)
- addElement('versioncode', str(apk['versioncode']), doc, apkel)
- addElement('apkname', apk['apkname'], doc, apkel)
- if apk.has_key('srcname'):
- addElement('srcname', apk['srcname'], doc, apkel)
- for hash_type in ('sha256', 'md5'):
- if not hash_type in apk:
- continue
- hashel = doc.createElement("hash")
- hashel.setAttribute("type", hash_type)
- hashel.appendChild(doc.createTextNode(apk[hash_type]))
- apkel.appendChild(hashel)
- addElement('sig', apk['sig'], doc, apkel)
- addElement('size', str(apk['size']), doc, apkel)
- addElement('sdkver', str(apk['sdkversion']), doc, apkel)
- perms = ""
- for p in apk['permissions']:
- if len(perms) > 0:
- perms += ","
- perms += p
- if len(perms) > 0:
- addElement('permissions', perms, doc, apkel)
- features = ""
- for f in apk['features']:
- if len(features) > 0:
- features += ","
- features += f
- if len(features) > 0:
- addElement('features', features, doc, apkel)
-
- if options.buildreport:
- if len(app['builds']) == 0:
- print ("WARNING: No builds defined for " + app['id'] +
- " Source: " + app['Source Code'])
- warnings += 1
+ # Generate warnings for apk's with no metadata (or create skeleton
+ # metadata files, if requested on the command line)
+ for apk in apks:
+ found = False
+ for app in apps:
+ if app['id'] == apk['id']:
+ found = True
+ break
+ if not found:
+ if options.createmeta:
+ f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
+ f.write("License:Unknown\n")
+ f.write("Web Site:\n")
+ f.write("Source Code:\n")
+ f.write("Issue Tracker:\n")
+ f.write("Summary:" + apk['name'] + "\n")
+ f.write("Description:\n")
+ f.write(apk['name'] + "\n")
+ f.write(".\n")
+ f.close()
+ print "Generated skeleton metadata for " + apk['id']
else:
- if app['Current Version Code'] != '0':
- gotbuild = False
- for build in app['builds']:
- if build['vercode'] == app['Current Version Code']:
- gotbuild = True
- if not gotbuild:
- print ("WARNING: No build data for current version of "
- + app['id'] + " (" + app['Current Version']
- + ") " + app['Source Code'])
- warnings += 1
+ print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
+ print " " + apk['name'] + " - " + apk['version']
- # If we don't have the current version, check if there is a build
- # with a commit ID starting with '!' - this means we can't build it
- # for some reason, and don't want hassling about it...
- if not gotcurrentver and app['Current Version Code'] != '0':
- for build in app['builds']:
- if build['vercode'] == app['Current Version Code']:
- gotcurrentver = True
+ #Sort the app list by name, then the web site doesn't have to by default:
+ apps = sorted(apps, key=lambda app: app['Name'].upper())
- # Output a message of harassment if we don't have the current version:
- if not gotcurrentver and app['Current Version Code'] != '0':
- addr = app['Source Code']
- print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name']
- print " (" + app['id'] + ") " + addr
- warnings += 1
- if options.verbose:
- # A bit of extra debug info, basically for diagnosing
- # app developer mistakes:
- print " Current vercode:" + app['Current Version Code']
- print " Got:"
- for apk in apks:
- if apk['id'] == app['id']:
- print " " + str(apk['versioncode']) + " - " + apk['version']
- if options.interactive:
- print "Build data out of date for " + app['id']
- while True:
- answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower()
- if answer == 'i':
- break
- elif answer == 'e':
- subprocess.call([options.editor,
- os.path.join('metadata',
- app['id'] + '.txt')])
- break
- elif answer == 'q':
- sys.exit(0)
- else:
- apps_disabled += 1
+ # Create the index
+ doc = Document()
-of = open(os.path.join('repo','index.xml'), 'wb')
-if options.pretty:
- output = doc.toprettyxml()
-else:
- output = doc.toxml()
-of.write(output)
-of.close()
+ def addElement(name, value, doc, parent):
+ el = doc.createElement(name)
+ el.appendChild(doc.createTextNode(value))
+ parent.appendChild(el)
-if repo_keyalias != None:
+ root = doc.createElement("fdroid")
+ doc.appendChild(root)
- if not options.quiet:
- print "Creating signed index."
- print "Key fingerprint:", repo_pubkey_fingerprint
-
- #Create a jar of the index...
- p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
- cwd='repo', stdout=subprocess.PIPE)
- output = p.communicate()[0]
- if options.verbose:
- print output
- if p.returncode != 0:
- print "ERROR: Failed to create jar file"
- sys.exit(1)
+ repoel = doc.createElement("repo")
+ repoel.setAttribute("name", repo_name)
+ repoel.setAttribute("icon", os.path.basename(repo_icon))
+ repoel.setAttribute("url", repo_url)
- # Sign the index...
- p = subprocess.Popen(['jarsigner', '-keystore', keystore,
- '-storepass', keystorepass, '-keypass', keypass,
- os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE)
- output = p.communicate()[0]
- if p.returncode != 0:
- print "Failed to sign index"
- print output
- sys.exit(1)
- if options.verbose:
- print output
+ if repo_keyalias != None:
-# Copy the repo icon into the repo directory...
-iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
-shutil.copyfile(repo_icon, iconfilename)
+ # Generate a certificate fingerprint the same way keytool does it
+ # (but with slightly different formatting)
+ def cert_fingerprint(data):
+ digest = hashlib.sha1(data).digest()
+ ret = []
+ for i in range(4):
+ ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
+ return " ".join(ret)
-# Write a category list in the repo to allow quick access...
-catdata = ''
-for cat in categories:
- catdata += cat + '\n'
-f = open('repo/categories.txt', 'w')
-f.write(catdata)
-f.close()
+ def extract_pubkey():
+ p = subprocess.Popen(['keytool', '-exportcert',
+ '-alias', repo_keyalias,
+ '-keystore', keystore,
+ '-storepass', keystorepass],
+ stdout=subprocess.PIPE)
+ cert = p.communicate()[0]
+ if p.returncode != 0:
+ print "ERROR: Failed to get repo pubkey"
+ sys.exit(1)
+ global repo_pubkey_fingerprint
+ repo_pubkey_fingerprint = cert_fingerprint(cert)
+ return "".join("%02x" % ord(b) for b in cert)
-# Update known apks info...
-knownapks = common.KnownApks()
-for apk in apks:
- knownapks.recordapk(apk['apkname'], apk['id'])
-knownapks.writeifchanged()
+ repoel.setAttribute("pubkey", extract_pubkey())
+
+ addElement('description', repo_description, doc, repoel)
+ root.appendChild(repoel)
+
+ apps_inrepo = 0
+ apps_disabled = 0
+ apps_nopkg = 0
-# Generate latest apps data for widget
-data = ''
-for line in file(os.path.join('stats', 'latestapps.txt')):
- appid = line.rstrip()
- data += appid + "\t"
for app in apps:
- if app['id'] == appid:
- data += app['Name'] + "\t"
- data += app['icon'] + "\t"
- data += app['License'] + "\n"
- break
-f = open('repo/latestapps.dat', 'w')
-f.write(data)
-f.close()
+
+ if app['Disabled'] is None:
+
+ # Get a list of the apks for this app...
+ gotcurrentver = False
+ apklist = []
+ for apk in apks:
+ if apk['id'] == app['id']:
+ if str(apk['versioncode']) == app['Current Version Code']:
+ gotcurrentver = True
+ apklist.append(apk)
+
+ if len(apklist) == 0:
+ apps_nopkg += 1
+ else:
+ apps_inrepo += 1
+ apel = doc.createElement("application")
+ apel.setAttribute("id", app['id'])
+ root.appendChild(apel)
+
+ addElement('id', app['id'], doc, apel)
+ addElement('name', app['Name'], doc, apel)
+ addElement('summary', app['Summary'], doc, apel)
+ addElement('icon', app['icon'], doc, apel)
+ addElement('description',
+ common.parse_description(app['Description']), doc, apel)
+ addElement('license', app['License'], doc, apel)
+ if 'Category' in app:
+ addElement('category', app['Category'], doc, apel)
+ addElement('web', app['Web Site'], doc, apel)
+ addElement('source', app['Source Code'], doc, apel)
+ addElement('tracker', app['Issue Tracker'], doc, apel)
+ if app['Donate'] != None:
+ addElement('donate', app['Donate'], doc, apel)
+
+ # These elements actually refer to the current version (i.e. which
+ # one is recommended. They are historically mis-named, and need
+ # changing, but stay like this for now to support existing clients.
+ addElement('marketversion', app['Current Version'], doc, apel)
+ addElement('marketvercode', app['Current Version Code'], doc, apel)
+
+ if not (app['AntiFeatures'] is None):
+ addElement('antifeatures', app['AntiFeatures'], doc, apel)
+ if app['Requires Root']:
+ addElement('requirements', 'root', doc, apel)
+
+ # Sort the apk list into version order, just so the web site
+ # doesn't have to do any work by default...
+ apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
+
+ # Check for duplicates - they will make the client unhappy...
+ for i in range(len(apklist) - 1):
+ if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
+ print "ERROR - duplicate versions"
+ print apklist[i]['apkname']
+ print apklist[i+1]['apkname']
+ sys.exit(1)
+
+ for apk in apklist:
+ apkel = doc.createElement("package")
+ apel.appendChild(apkel)
+ addElement('version', apk['version'], doc, apkel)
+ addElement('versioncode', str(apk['versioncode']), doc, apkel)
+ addElement('apkname', apk['apkname'], doc, apkel)
+ if apk.has_key('srcname'):
+ addElement('srcname', apk['srcname'], doc, apkel)
+ for hash_type in ('sha256', 'md5'):
+ if not hash_type in apk:
+ continue
+ hashel = doc.createElement("hash")
+ hashel.setAttribute("type", hash_type)
+ hashel.appendChild(doc.createTextNode(apk[hash_type]))
+ apkel.appendChild(hashel)
+ addElement('sig', apk['sig'], doc, apkel)
+ addElement('size', str(apk['size']), doc, apkel)
+ addElement('sdkver', str(apk['sdkversion']), doc, apkel)
+ perms = ""
+ for p in apk['permissions']:
+ if len(perms) > 0:
+ perms += ","
+ perms += p
+ if len(perms) > 0:
+ addElement('permissions', perms, doc, apkel)
+ features = ""
+ for f in apk['features']:
+ if len(features) > 0:
+ features += ","
+ features += f
+ if len(features) > 0:
+ addElement('features', features, doc, apkel)
+
+ if options.buildreport:
+ if len(app['builds']) == 0:
+ print ("WARNING: No builds defined for " + app['id'] +
+ " Source: " + app['Source Code'])
+ warnings += 1
+ else:
+ if app['Current Version Code'] != '0':
+ gotbuild = False
+ for build in app['builds']:
+ if build['vercode'] == app['Current Version Code']:
+ gotbuild = True
+ if not gotbuild:
+ print ("WARNING: No build data for current version of "
+ + app['id'] + " (" + app['Current Version']
+ + ") " + app['Source Code'])
+ warnings += 1
+
+ # If we don't have the current version, check if there is a build
+ # with a commit ID starting with '!' - this means we can't build it
+ # for some reason, and don't want hassling about it...
+ if not gotcurrentver and app['Current Version Code'] != '0':
+ for build in app['builds']:
+ if build['vercode'] == app['Current Version Code']:
+ gotcurrentver = True
+
+ # Output a message of harassment if we don't have the current version:
+ if not gotcurrentver and app['Current Version Code'] != '0':
+ addr = app['Source Code']
+ print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name']
+ print " (" + app['id'] + ") " + addr
+ warnings += 1
+ if options.verbose:
+ # A bit of extra debug info, basically for diagnosing
+ # app developer mistakes:
+ print " Current vercode:" + app['Current Version Code']
+ print " Got:"
+ for apk in apks:
+ if apk['id'] == app['id']:
+ print " " + str(apk['versioncode']) + " - " + apk['version']
+ if options.interactive:
+ print "Build data out of date for " + app['id']
+ while True:
+ answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower()
+ if answer == 'i':
+ break
+ elif answer == 'e':
+ subprocess.call([options.editor,
+ os.path.join('metadata',
+ app['id'] + '.txt')])
+ break
+ elif answer == 'q':
+ sys.exit(0)
+ else:
+ apps_disabled += 1
+
+ of = open(os.path.join('repo','index.xml'), 'wb')
+ if options.pretty:
+ output = doc.toprettyxml()
+ else:
+ output = doc.toxml()
+ of.write(output)
+ of.close()
+
+ if repo_keyalias != None:
+
+ if not options.quiet:
+ print "Creating signed index."
+ print "Key fingerprint:", repo_pubkey_fingerprint
+
+ #Create a jar of the index...
+ p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
+ cwd='repo', stdout=subprocess.PIPE)
+ output = p.communicate()[0]
+ if options.verbose:
+ print output
+ if p.returncode != 0:
+ print "ERROR: Failed to create jar file"
+ sys.exit(1)
+
+ # Sign the index...
+ p = subprocess.Popen(['jarsigner', '-keystore', keystore,
+ '-storepass', keystorepass, '-keypass', keypass,
+ os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE)
+ output = p.communicate()[0]
+ if p.returncode != 0:
+ print "Failed to sign index"
+ print output
+ sys.exit(1)
+ if options.verbose:
+ print output
+
+ # Copy the repo icon into the repo directory...
+ iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
+ shutil.copyfile(repo_icon, iconfilename)
+
+ # Write a category list in the repo to allow quick access...
+ catdata = ''
+ for cat in categories:
+ catdata += cat + '\n'
+ f = open('repo/categories.txt', 'w')
+ f.write(catdata)
+ f.close()
+
+ # Update known apks info...
+ knownapks = common.KnownApks()
+ for apk in apks:
+ knownapks.recordapk(apk['apkname'], apk['id'])
+ knownapks.writeifchanged()
+
+ # Generate latest apps data for widget
+ data = ''
+ for line in file(os.path.join('stats', 'latestapps.txt')):
+ appid = line.rstrip()
+ data += appid + "\t"
+ for app in apps:
+ if app['id'] == appid:
+ data += app['Name'] + "\t"
+ data += app['icon'] + "\t"
+ data += app['License'] + "\n"
+ break
+ f = open('repo/latestapps.dat', 'w')
+ f.write(data)
+ f.close()
-print "Finished."
-print str(apps_inrepo) + " apps in repo"
-print str(apps_disabled) + " disabled"
-print str(apps_nopkg) + " with no packages"
-print str(warnings) + " warnings"
+ print "Finished."
+ print str(apps_inrepo) + " apps in repo"
+ print str(apps_disabled) + " disabled"
+ print str(apps_nopkg) + " with no packages"
+ print str(warnings) + " warnings"
+
+if __name__ == "__main__":
+ main()
diff --git a/updatestats.py b/updatestats.py
index 892c20bc..a6f98169 100755
--- a/updatestats.py
+++ b/updatestats.py
@@ -30,121 +30,125 @@ import HTMLParser
import paramiko
import common
-#Read configuration...
-execfile('config.py')
+def main():
+
+ # Read configuration...
+ execfile('config.py')
+
+ # Parse command line...
+ parser = OptionParser()
+ parser.add_option("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_option("-d", "--download", action="store_true", default=False,
+ help="Download logs we don't have")
+ (options, args) = parser.parse_args()
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
- help="Spew out even more information than normal")
-parser.add_option("-d", "--download", action="store_true", default=False,
- help="Download logs we don't have")
-(options, args) = parser.parse_args()
+ statsdir = 'stats'
+ logsdir = os.path.join(statsdir, 'logs')
+ logsarchivedir = os.path.join(logsdir, 'archive')
+ datadir = os.path.join(statsdir, 'data')
+ if not os.path.exists(statsdir):
+ os.mkdir(statsdir)
+ if not os.path.exists(logsdir):
+ os.mkdir(logsdir)
+ if not os.path.exists(datadir):
+ os.mkdir(datadir)
+ if options.download:
+ # Get any access logs we don't have...
+ ssh = None
+ ftp = None
+ try:
+ print 'Retrieving logs'
+ ssh = paramiko.SSHClient()
+ ssh.load_system_host_keys()
+ ssh.connect('f-droid.org', username='fdroid', timeout=10,
+ key_filename=webserver_keyfile)
+ ftp = ssh.open_sftp()
+ ftp.get_channel().settimeout(15)
+ print "...connected"
-statsdir = 'stats'
-logsdir = os.path.join(statsdir, 'logs')
-logsarchivedir = os.path.join(logsdir, 'archive')
-datadir = os.path.join(statsdir, 'data')
-if not os.path.exists(statsdir):
- os.mkdir(statsdir)
-if not os.path.exists(logsdir):
- os.mkdir(logsdir)
-if not os.path.exists(datadir):
- os.mkdir(datadir)
+ ftp.chdir('logs')
+ files = ftp.listdir()
+ for f in files:
+ if f.startswith('access-') and f.endswith('.log'):
-if options.download:
- # Get any access logs we don't have...
- ssh = None
- ftp = None
- try:
- print 'Retrieving logs'
- ssh = paramiko.SSHClient()
- ssh.load_system_host_keys()
- ssh.connect('f-droid.org', username='fdroid', timeout=10,
- key_filename=webserver_keyfile)
- ftp = ssh.open_sftp()
- ftp.get_channel().settimeout(15)
- print "...connected"
-
- ftp.chdir('logs')
- files = ftp.listdir()
- for f in files:
- if f.startswith('access-') and f.endswith('.log'):
-
- destpath = os.path.join(logsdir, f)
- archivepath = os.path.join(logsarchivedir, f + '.gz')
- if os.path.exists(archivepath):
- if os.path.exists(destpath):
- # Just in case we have it archived but failed to remove
- # the original...
- os.remove(destpath)
- else:
- destsize = ftp.stat(f).st_size
- if (not os.path.exists(destpath) or
- os.path.getsize(destpath) != destsize):
- print "...retrieving " + f
- ftp.get(f, destpath)
- except Exception as e:
- traceback.print_exc()
- sys.exit(1)
- finally:
- #Disconnect
- if ftp != None:
- ftp.close()
- if ssh != None:
- ssh.close()
-
-# Process logs
-logexpr = '(?P[.:0-9a-fA-F]+) - - \[(?P