From 00abc9527d2930358197b42e51c5fb3b4aa96d81 Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Sun, 26 Feb 2012 14:18:58 +0000 Subject: [PATCH] All callable scripts now implement main() --- checkupdates.py | 80 +++-- import.py | 386 +++++++++++---------- publish.py | 174 +++++----- rewritemeta.py | 30 +- scanner.py | 119 ++++--- update.py | 899 ++++++++++++++++++++++++------------------------ updatestats.py | 218 ++++++------ 7 files changed, 970 insertions(+), 936 deletions(-) diff --git a/checkupdates.py b/checkupdates.py index 2c083044..b679d6f8 100755 --- a/checkupdates.py +++ b/checkupdates.py @@ -27,9 +27,6 @@ from optparse import OptionParser import HTMLParser import common -#Read configuration... -execfile('config.py') - # Check for a new version by looking at the Google market. # Returns (None, "a message") if this didn't work, or (version, vercode) for @@ -66,48 +63,55 @@ def check_market(app): +def main(): -# Parse command line... -parser = OptionParser() -parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") -parser.add_option("-p", "--package", default=None, - help="Build only the specified package") -(options, args) = parser.parse_args() + #Read configuration... + execfile('config.py') -# Get all apps... -apps = common.read_metadata(options.verbose) + # Parse command line... + parser = OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + parser.add_option("-p", "--package", default=None, + help="Build only the specified package") + (options, args) = parser.parse_args() -html_parser = HTMLParser.HTMLParser() + # Get all apps... + apps = common.read_metadata(options.verbose) -for app in apps: + html_parser = HTMLParser.HTMLParser() - if options.package and options.package != app['id']: - # Silent skip... - pass - else: - print "Processing " + app['id'] + '...' + for app in apps: - mode = app['Update Check Mode'] - if mode == 'Market': - (version, vercode) = check_market(app) - elif mode == 'None': - version = None - vercode = 'Checking disabled' + if options.package and options.package != app['id']: + # Silent skip... + pass else: - version = None - vercode = 'Invalid update check method' + print "Processing " + app['id'] + '...' - if not version: - print "..." + vercode - elif vercode == app['Current Version Code'] and version == app['Current Version']: - print "...up to date" - else: - print '...updating to version:' + version + ' vercode:' + vercode - app['Current Version'] = version - app['Current Version Code'] = vercode - metafile = os.path.join('metadata', app['id'] + '.txt') - common.write_metadata(metafile, app) + mode = app['Update Check Mode'] + if mode == 'Market': + (version, vercode) = check_market(app) + elif mode == 'None': + version = None + vercode = 'Checking disabled' + else: + version = None + vercode = 'Invalid update check method' -print "Finished." + if not version: + print "..." + vercode + elif vercode == app['Current Version Code'] and version == app['Current Version']: + print "...up to date" + else: + print '...updating to version:' + version + ' vercode:' + vercode + app['Current Version'] = version + app['Current Version Code'] = vercode + metafile = os.path.join('metadata', app['id'] + '.txt') + common.write_metadata(metafile, app) + + print "Finished." + +if __name__ == "__main__": + main() diff --git a/import.py b/import.py index e29fbc3a..3d794cb8 100755 --- a/import.py +++ b/import.py @@ -25,219 +25,225 @@ import re import urllib from optparse import OptionParser -#Read configuration... -repo_name = None -repo_description = None -repo_icon = None -repo_url = None -execfile('config.py') +def main(): -import common + # Read configuration... + repo_name = None + repo_description = None + repo_icon = None + repo_url = None + execfile('config.py') -# Parse command line... -parser = OptionParser() -parser.add_option("-u", "--url", default=None, - help="Project URL to import from.") -parser.add_option("-s", "--subdir", default=None, - help="Path to main android project subdirectory, if not in root.") -(options, args) = parser.parse_args() + import common -if not options.url: - print "Specify project url." - sys.exit(1) -url = options.url + # Parse command line... + parser = OptionParser() + parser.add_option("-u", "--url", default=None, + help="Project URL to import from.") + parser.add_option("-s", "--subdir", default=None, + help="Path to main android project subdirectory, if not in root.") + (options, args) = parser.parse_args() -tmp_dir = 'tmp' -if not os.path.isdir(tmp_dir): - print "Creating temporary directory" - os.makedirs(tmp_dir) - -# Get all apps... -apps = common.read_metadata() - -# Figure out what kind of project it is... -projecttype = None -issuetracker = None -license = None -if url.startswith('https://github.com'): - projecttype = 'github' - repo = url + '.git' - repotype = 'git' - sourcecode = url -elif url.startswith('http://code.google.com/p/'): - if not url.endswith('/'): - print "Expected format for googlecode url is http://code.google.com/p/PROJECT/" + if not options.url: + print "Specify project url." sys.exit(1) - projecttype = 'googlecode' - sourcecode = url + 'source/checkout' - issuetracker = url + 'issues/list' + url = options.url - # Figure out the repo type and adddress... - req = urllib.urlopen(sourcecode) - if req.getcode() != 200: - print 'Unable to find source at ' + sourcecode + ' - return code ' + str(req.getcode()) - sys.exit(1) - page = req.read() - repotype = None - index = page.find('hg clone') - if index != -1: - repotype = 'hg' - repo = page[index + 9:] - index = repo.find('<') - if index == -1: - print "Error while getting repo address" + tmp_dir = 'tmp' + if not os.path.isdir(tmp_dir): + print "Creating temporary directory" + os.makedirs(tmp_dir) + + # Get all apps... + apps = common.read_metadata() + + # Figure out what kind of project it is... + projecttype = None + issuetracker = None + license = None + if url.startswith('https://github.com'): + projecttype = 'github' + repo = url + '.git' + repotype = 'git' + sourcecode = url + elif url.startswith('http://code.google.com/p/'): + if not url.endswith('/'): + print "Expected format for googlecode url is http://code.google.com/p/PROJECT/" sys.exit(1) - repo = repo[:index] - if not repotype: - index=page.find('git clone') + projecttype = 'googlecode' + sourcecode = url + 'source/checkout' + issuetracker = url + 'issues/list' + + # Figure out the repo type and adddress... + req = urllib.urlopen(sourcecode) + if req.getcode() != 200: + print 'Unable to find source at ' + sourcecode + ' - return code ' + str(req.getcode()) + sys.exit(1) + page = req.read() + repotype = None + index = page.find('hg clone') if index != -1: - repotype = 'git' - repo = page[index + 10:] + repotype = 'hg' + repo = page[index + 9:] index = repo.find('<') if index == -1: print "Error while getting repo address" sys.exit(1) repo = repo[:index] - if not repotype: - index=page.find('svn checkout') - if index != -1: - repotype = 'git-svn' - repo = page[index + 13:] - prefix = 'http' - if not repo.startswith(prefix): - print "Unexpected checkout instructions format" - sys.exit(1) - repo = 'http' + repo[len(prefix):] - index = repo.find('<') - if index == -1: - print "Error while getting repo address - no end tag? '" + repo + "'" - sys.exit(1) - repo = repo[:index] - index = repo.find(' ') - if index == -1: - print "Error while getting repo address - no space? '" + repo + "'" - sys.exit(1) - repo = repo[:index] - if not repotype: - print "Unable to determine vcs type" + if not repotype: + index=page.find('git clone') + if index != -1: + repotype = 'git' + repo = page[index + 10:] + index = repo.find('<') + if index == -1: + print "Error while getting repo address" + sys.exit(1) + repo = repo[:index] + if not repotype: + index=page.find('svn checkout') + if index != -1: + repotype = 'git-svn' + repo = page[index + 13:] + prefix = 'http' + if not repo.startswith(prefix): + print "Unexpected checkout instructions format" + sys.exit(1) + repo = 'http' + repo[len(prefix):] + index = repo.find('<') + if index == -1: + print "Error while getting repo address - no end tag? '" + repo + "'" + sys.exit(1) + repo = repo[:index] + index = repo.find(' ') + if index == -1: + print "Error while getting repo address - no space? '" + repo + "'" + sys.exit(1) + repo = repo[:index] + if not repotype: + print "Unable to determine vcs type" + sys.exit(1) + + # Figure out the license... + req = urllib.urlopen(url) + if req.getcode() != 200: + print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()) + sys.exit(1) + page = req.read() + index = page.find('Code license') + if index == -1: + print "Couldn't find license data" + sys.exit(1) + ltext = page[index:] + lprefix = 'rel="nofollow">' + index = ltext.find(lprefix) + if index == -1: + print "Couldn't find license text" + sys.exit(1) + ltext = ltext[index + len(lprefix):] + index = ltext.find('<') + if index == -1: + print "License text not formatted as expected" + sys.exit(1) + ltext = ltext[:index] + if ltext == 'GNU GPL v3': + license = 'GPLv3' + elif ltext == 'GNU GPL v2': + license = 'GPLv2' + elif ltext == 'Apache License 2.0': + license = 'Apache2' + else: + print "License " + ltext + " is not recognised" + sys.exit(1) + + if not projecttype: + print "Unable to determine the project type." sys.exit(1) - # Figure out the license... - req = urllib.urlopen(url) - if req.getcode() != 200: - print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()) - sys.exit(1) - page = req.read() - index = page.find('Code license') - if index == -1: - print "Couldn't find license data" - sys.exit(1) - ltext = page[index:] - lprefix = 'rel="nofollow">' - index = ltext.find(lprefix) - if index == -1: - print "Couldn't find license text" - sys.exit(1) - ltext = ltext[index + len(lprefix):] - index = ltext.find('<') - if index == -1: - print "License text not formatted as expected" - sys.exit(1) - ltext = ltext[:index] - if ltext == 'GNU GPL v3': - license = 'GPLv3' - elif ltext == 'GNU GPL v2': - license = 'GPLv2' - elif ltext == 'Apache License 2.0': - license = 'Apache2' + # Get a copy of the source so we can extract some info... + print 'Getting source from ' + repotype + ' repo at ' + repo + src_dir = os.path.join(tmp_dir, 'importer') + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + vcs = common.getvcs(repotype, repo, src_dir) + vcs.gotorevision(None) + if options.subdir: + root_dir = os.path.join(src_dir, options.subdir) else: - print "License " + ltext + " is not recognised" + root_dir = src_dir + + # Check AndroidManiifest.xml exists... + manifest = os.path.join(root_dir, 'AndroidManifest.xml') + if not os.path.exists(manifest): + print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?" sys.exit(1) -if not projecttype: - print "Unable to determine the project type." - sys.exit(1) - -# Get a copy of the source so we can extract some info... -print 'Getting source from ' + repotype + ' repo at ' + repo -src_dir = os.path.join(tmp_dir, 'importer') -if os.path.exists(tmp_dir): - shutil.rmtree(tmp_dir) -vcs = common.getvcs(repotype, repo, src_dir) -vcs.gotorevision(None) -if options.subdir: - root_dir = os.path.join(src_dir, options.subdir) -else: - root_dir = src_dir - -# Check AndroidManiifest.xml exists... -manifest = os.path.join(root_dir, 'AndroidManifest.xml') -if not os.path.exists(manifest): - print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?" - sys.exit(1) - -# Extract some information... -vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search -vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search -psearch = re.compile(r'.*package="([^"]+)".*').search -version = None -vercode = None -package = None -for line in file(manifest): + # Extract some information... + vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search + vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search + psearch = re.compile(r'.*package="([^"]+)".*').search + version = None + vercode = None + package = None + for line in file(manifest): + if not package: + matches = psearch(line) + if matches: + package = matches.group(1) + if not version: + matches = vnsearch(line) + if matches: + version = matches.group(1) + if not vercode: + matches = vcsearch(line) + if matches: + vercode = matches.group(1) if not package: - matches = psearch(line) - if matches: - package = matches.group(1) + print "Couldn't find package ID" + sys.exit(1) if not version: - matches = vnsearch(line) - if matches: - version = matches.group(1) + print "Couldn't find latest version name" + sys.exit(1) if not vercode: - matches = vcsearch(line) - if matches: - vercode = matches.group(1) -if not package: - print "Couldn't find package ID" - sys.exit(1) -if not version: - print "Couldn't find latest version name" - sys.exit(1) -if not vercode: - print "Couldn't find latest version code" - sys.exit(1) - -# Make sure it's actually new... -for app in apps: - if app['id'] == package: - print "Package " + package + " already exists" + print "Couldn't find latest version code" sys.exit(1) -# Construct the metadata... -app = common.parse_metadata(None) -app['id'] = package -app['Web Site'] = url -app['Source Code'] = sourcecode -if issuetracker: - app['Issue Tracker'] = issuetracker -if license: - app['License'] = license -app['Repo Type'] = repotype -app['Repo'] = repo + # Make sure it's actually new... + for app in apps: + if app['id'] == package: + print "Package " + package + " already exists" + sys.exit(1) -# Create a build line... -build = {} -build['version'] = version -build['vercode'] = vercode -build['commit'] = '?' -if options.subdir: - build['subdir'] = options.subdir -if os.path.exists(os.path.join(root_dir, 'jni')): - build['buildjni'] = 'yes' -app['builds'].append(build) -app['comments'].append(('build:' + version, - "#Generated by import.py - check this is the right version, and find the right commit!")) + # Construct the metadata... + app = common.parse_metadata(None) + app['id'] = package + app['Web Site'] = url + app['Source Code'] = sourcecode + if issuetracker: + app['Issue Tracker'] = issuetracker + if license: + app['License'] = license + app['Repo Type'] = repotype + app['Repo'] = repo -metafile = os.path.join('metadata', package + '.txt') -common.write_metadata(metafile, app) -print "Wrote " + metafile + # Create a build line... + build = {} + build['version'] = version + build['vercode'] = vercode + build['commit'] = '?' + if options.subdir: + build['subdir'] = options.subdir + if os.path.exists(os.path.join(root_dir, 'jni')): + build['buildjni'] = 'yes' + app['builds'].append(build) + app['comments'].append(('build:' + version, + "#Generated by import.py - check this is the right version, and find the right commit!")) + + metafile = os.path.join('metadata', package + '.txt') + common.write_metadata(metafile, app) + print "Wrote " + metafile + + +if __name__ == "__main__": + main() diff --git a/publish.py b/publish.py index 7fd9a1fe..c5610ffe 100755 --- a/publish.py +++ b/publish.py @@ -31,106 +31,112 @@ from optparse import OptionParser import common from common import BuildException -#Read configuration... -execfile('config.py') +def main(): -# Parse command line... -parser = OptionParser() -parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") -parser.add_option("-p", "--package", default=None, - help="Publish only the specified package") -(options, args) = parser.parse_args() + #Read configuration... + execfile('config.py') -log_dir = 'logs' -if not os.path.isdir(log_dir): - print "Creating log directory" - os.makedirs(log_dir) + # Parse command line... + parser = OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + parser.add_option("-p", "--package", default=None, + help="Publish only the specified package") + (options, args) = parser.parse_args() -tmp_dir = 'tmp' -if not os.path.isdir(tmp_dir): - print "Creating temporary directory" - os.makedirs(tmp_dir) + log_dir = 'logs' + if not os.path.isdir(log_dir): + print "Creating log directory" + os.makedirs(log_dir) -output_dir = 'repo' -if not os.path.isdir(output_dir): - print "Creating output directory" - os.makedirs(output_dir) + tmp_dir = 'tmp' + if not os.path.isdir(tmp_dir): + print "Creating temporary directory" + os.makedirs(tmp_dir) -unsigned_dir = 'unsigned' -if not os.path.isdir(unsigned_dir): - print "No unsigned directory - nothing to do" - sys.exit(0) + output_dir = 'repo' + if not os.path.isdir(output_dir): + print "Creating output directory" + os.makedirs(output_dir) -for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))): + unsigned_dir = 'unsigned' + if not os.path.isdir(unsigned_dir): + print "No unsigned directory - nothing to do" + sys.exit(0) - apkfilename = os.path.basename(apkfile) - i = apkfilename.rfind('_') - if i == -1: - raise BuildException("Invalid apk name") - appid = apkfilename[:i] - print "Processing " + appid + for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))): - if not options.package or options.package == appid: + apkfilename = os.path.basename(apkfile) + i = apkfilename.rfind('_') + if i == -1: + raise BuildException("Invalid apk name") + appid = apkfilename[:i] + print "Processing " + appid - # Figure out the key alias name we'll use. Only the first 8 - # characters are significant, so we'll use the first 8 from - # the MD5 of the app's ID and hope there are no collisions. - # If a collision does occur later, we're going to have to - # come up with a new alogrithm, AND rename all existing keys - # in the keystore! - if keyaliases.has_key(appid): - # For this particular app, the key alias is overridden... - keyalias = keyaliases[appid] - else: - m = md5.new() - m.update(appid) - keyalias = m.hexdigest()[:8] - print "Key alias: " + keyalias + if not options.package or options.package == appid: - # See if we already have a key for this application, and - # if not generate one... - p = subprocess.Popen(['keytool', '-list', - '-alias', keyalias, '-keystore', keystore, - '-storepass', keystorepass], stdout=subprocess.PIPE) - output = p.communicate()[0] - if p.returncode !=0: - print "Key does not exist - generating..." - p = subprocess.Popen(['keytool', '-genkey', - '-keystore', keystore, '-alias', keyalias, - '-keyalg', 'RSA', '-keysize', '2048', - '-validity', '10000', + # Figure out the key alias name we'll use. Only the first 8 + # characters are significant, so we'll use the first 8 from + # the MD5 of the app's ID and hope there are no collisions. + # If a collision does occur later, we're going to have to + # come up with a new alogrithm, AND rename all existing keys + # in the keystore! + if keyaliases.has_key(appid): + # For this particular app, the key alias is overridden... + keyalias = keyaliases[appid] + else: + m = md5.new() + m.update(appid) + keyalias = m.hexdigest()[:8] + print "Key alias: " + keyalias + + # See if we already have a key for this application, and + # if not generate one... + p = subprocess.Popen(['keytool', '-list', + '-alias', keyalias, '-keystore', keystore, + '-storepass', keystorepass], stdout=subprocess.PIPE) + output = p.communicate()[0] + if p.returncode !=0: + print "Key does not exist - generating..." + p = subprocess.Popen(['keytool', '-genkey', + '-keystore', keystore, '-alias', keyalias, + '-keyalg', 'RSA', '-keysize', '2048', + '-validity', '10000', + '-storepass', keystorepass, '-keypass', keypass, + '-dname', keydname], stdout=subprocess.PIPE) + output = p.communicate()[0] + print output + if p.returncode != 0: + raise BuildException("Failed to generate key") + + # Sign the application... + p = subprocess.Popen(['jarsigner', '-keystore', keystore, '-storepass', keystorepass, '-keypass', keypass, - '-dname', keydname], stdout=subprocess.PIPE) + apkfile, keyalias], stdout=subprocess.PIPE) output = p.communicate()[0] print output if p.returncode != 0: - raise BuildException("Failed to generate key") + raise BuildException("Failed to sign application") - # Sign the application... - p = subprocess.Popen(['jarsigner', '-keystore', keystore, - '-storepass', keystorepass, '-keypass', keypass, - apkfile, keyalias], stdout=subprocess.PIPE) - output = p.communicate()[0] - print output - if p.returncode != 0: - raise BuildException("Failed to sign application") + # Zipalign it... + p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'), + '-v', '4', apkfile, + os.path.join(output_dir, apkfilename)], + stdout=subprocess.PIPE) + output = p.communicate()[0] + print output + if p.returncode != 0: + raise BuildException("Failed to align application") + os.remove(apkfile) - # Zipalign it... - p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'), - '-v', '4', apkfile, - os.path.join(output_dir, apkfilename)], - stdout=subprocess.PIPE) - output = p.communicate()[0] - print output - if p.returncode != 0: - raise BuildException("Failed to align application") - os.remove(apkfile) + # Move the source tarball into the output directory... + tarfilename = apkfilename[:-4] + '_src.tar.gz' + shutil.move(os.path.join(unsigned_dir, tarfilename), + os.path.join(output_dir, tarfilename)) - # Move the source tarball into the output directory... - tarfilename = apkfilename[:-4] + '_src.tar.gz' - shutil.move(os.path.join(unsigned_dir, tarfilename), - os.path.join(output_dir, tarfilename)) + print 'Published ' + apkfilename - print 'Published ' + apkfilename + +if __name__ == "__main__": + main() diff --git a/rewritemeta.py b/rewritemeta.py index e3535da2..b633f0cc 100755 --- a/rewritemeta.py +++ b/rewritemeta.py @@ -27,22 +27,26 @@ from optparse import OptionParser import HTMLParser import common -#Read configuration... -execfile('config.py') +def main(): + #Read configuration... + execfile('config.py') -# Parse command line... -parser = OptionParser() -parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") -(options, args) = parser.parse_args() + # Parse command line... + parser = OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (options, args) = parser.parse_args() -# Get all apps... -apps = common.read_metadata(options.verbose) + # Get all apps... + apps = common.read_metadata(options.verbose) -for app in apps: - print "Writing " + app['id'] - common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) + for app in apps: + print "Writing " + app['id'] + common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) -print "Finished." + print "Finished." + +if __name__ == "__main__": + main() diff --git a/scanner.py b/scanner.py index ee00a3ca..ad4e20fb 100755 --- a/scanner.py +++ b/scanner.py @@ -31,80 +31,85 @@ import common from common import BuildException from common import VCSException -#Read configuration... -execfile('config.py') +def main(): + + # Read configuration... + execfile('config.py') -# Parse command line... -parser = OptionParser() -parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") -parser.add_option("-p", "--package", default=None, - help="Scan only the specified package") -(options, args) = parser.parse_args() + # Parse command line... + parser = OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + parser.add_option("-p", "--package", default=None, + help="Scan only the specified package") + (options, args) = parser.parse_args() -# Get all apps... -apps = common.read_metadata(options.verbose) + # Get all apps... + apps = common.read_metadata(options.verbose) -html_parser = HTMLParser.HTMLParser() + html_parser = HTMLParser.HTMLParser() -problems = [] + problems = [] -extlib_dir = os.path.join('build', 'extlib') + extlib_dir = os.path.join('build', 'extlib') -for app in apps: + for app in apps: - skip = False - if options.package and app['id'] != options.package: - skip = True - elif app['Disabled']: - print "Skipping %s: disabled" % app['id'] - skip = True - elif not app['builds']: - print "Skipping %s: no builds specified" % app['id'] - skip = True + skip = False + if options.package and app['id'] != options.package: + skip = True + elif app['Disabled']: + print "Skipping %s: disabled" % app['id'] + skip = True + elif not app['builds']: + print "Skipping %s: no builds specified" % app['id'] + skip = True - if not skip: + if not skip: - print "Processing " + app['id'] + print "Processing " + app['id'] - try: + try: - build_dir = 'build/' + app['id'] + build_dir = 'build/' + app['id'] - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) - for thisbuild in app['builds']: + for thisbuild in app['builds']: - if thisbuild['commit'].startswith('!'): - print ("..skipping version " + thisbuild['version'] + " - " + - thisbuild['commit'][1:]) - else: - print "..scanning version " + thisbuild['version'] + if thisbuild['commit'].startswith('!'): + print ("..skipping version " + thisbuild['version'] + " - " + + thisbuild['commit'][1:]) + else: + print "..scanning version " + thisbuild['version'] - # Prepare the source code... - root_dir = common.prepare_source(vcs, app, thisbuild, - build_dir, extlib_dir, sdk_path, ndk_path, javacc_path) + # Prepare the source code... + root_dir = common.prepare_source(vcs, app, thisbuild, + build_dir, extlib_dir, sdk_path, ndk_path, javacc_path) - # Do the scan... - buildprobs = common.scan_source(build_dir, root_dir, thisbuild) - for problem in buildprobs: - problems.append(problem + - ' in ' + app['id'] + ' ' + thisbuild['version']) + # Do the scan... + buildprobs = common.scan_source(build_dir, root_dir, thisbuild) + for problem in buildprobs: + problems.append(problem + + ' in ' + app['id'] + ' ' + thisbuild['version']) - except BuildException as be: - msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be) - problems.append(msg) - except VCSException as vcse: - msg = "VCS error while scanning app %s: %s" % (app['id'], vcse) - problems.append(msg) - except Exception: - msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) - problems.append(msg) + except BuildException as be: + msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be) + problems.append(msg) + except VCSException as vcse: + msg = "VCS error while scanning app %s: %s" % (app['id'], vcse) + problems.append(msg) + except Exception: + msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) + problems.append(msg) -print "Finished:" -for problem in problems: - print problem -print str(len(problems)) + ' problems.' + print "Finished:" + for problem in problems: + print problem + print str(len(problems)) + ' problems.' + +if __name__ == "__main__": + main() diff --git a/update.py b/update.py index ad0f2377..f378db7c 100755 --- a/update.py +++ b/update.py @@ -29,491 +29,496 @@ from xml.dom.minidom import Document from optparse import OptionParser import time -#Read configuration... -repo_name = None -repo_description = None -repo_icon = None -repo_url = None -execfile('config.py') +def main(): -import common + # Read configuration... + repo_name = None + repo_description = None + repo_icon = None + repo_url = None + execfile('config.py') -# Parse command line... -parser = OptionParser() -parser.add_option("-c", "--createmeta", action="store_true", default=False, - help="Create skeleton metadata files that are missing") -parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") -parser.add_option("-q", "--quiet", action="store_true", default=False, - help="No output, except for warnings and errors") -parser.add_option("-b", "--buildreport", action="store_true", default=False, - help="Report on build data status") -parser.add_option("-i", "--interactive", default=False, action="store_true", - help="Interactively ask about things that need updating.") -parser.add_option("-e", "--editor", default="/etc/alternatives/editor", - help="Specify editor to use in interactive mode. Default "+ - "is /etc/alternatives/editor") -parser.add_option("", "--pretty", action="store_true", default=False, - help="Produce human-readable index.xml") -(options, args) = parser.parse_args() + import common + + # Parse command line... + parser = OptionParser() + parser.add_option("-c", "--createmeta", action="store_true", default=False, + help="Create skeleton metadata files that are missing") + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + parser.add_option("-q", "--quiet", action="store_true", default=False, + help="No output, except for warnings and errors") + parser.add_option("-b", "--buildreport", action="store_true", default=False, + help="Report on build data status") + parser.add_option("-i", "--interactive", default=False, action="store_true", + help="Interactively ask about things that need updating.") + parser.add_option("-e", "--editor", default="/etc/alternatives/editor", + help="Specify editor to use in interactive mode. Default "+ + "is /etc/alternatives/editor") + parser.add_option("", "--pretty", action="store_true", default=False, + help="Produce human-readable index.xml") + (options, args) = parser.parse_args() -icon_dir=os.path.join('repo','icons') + icon_dir=os.path.join('repo','icons') -# Delete and re-create the icon directory... -if os.path.exists(icon_dir): - shutil.rmtree(icon_dir) -os.mkdir(icon_dir) + # Delete and re-create the icon directory... + if os.path.exists(icon_dir): + shutil.rmtree(icon_dir) + os.mkdir(icon_dir) -warnings = 0 + warnings = 0 -#Make sure we have the repository description... -if (repo_url is None or repo_name is None or - repo_icon is None or repo_description is None): - print "Repository description fields are required in config.py" - print "See config.sample.py for details" - sys.exit(1) - -# Get all apps... -apps = common.read_metadata(verbose=options.verbose) - -# Generate a list of categories... -categories = [] -for app in apps: - if app['Category'] not in categories: - categories.append(app['Category']) - -# Gather information about all the apk files in the repo directory... -apks = [] -for apkfile in glob.glob(os.path.join('repo','*.apk')): - - apkfilename = apkfile[5:] - if apkfilename.find(' ') != -1: - print "No spaces in APK filenames!" + # Make sure we have the repository description... + if (repo_url is None or repo_name is None or + repo_icon is None or repo_description is None): + print "Repository description fields are required in config.py" + print "See config.sample.py for details" sys.exit(1) - srcfilename = apkfilename[:-4] + "_src.tar.gz" - if not options.quiet: - print "Processing " + apkfilename - thisinfo = {} - thisinfo['apkname'] = apkfilename - if os.path.exists(os.path.join('repo', srcfilename)): - thisinfo['srcname'] = srcfilename - thisinfo['size'] = os.path.getsize(apkfile) - thisinfo['permissions'] = [] - thisinfo['features'] = [] - p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'), - 'dump', 'badging', apkfile], - stdout=subprocess.PIPE) - output = p.communicate()[0] - if options.verbose: - print output - if p.returncode != 0: - print "ERROR: Failed to get apk information" - sys.exit(1) - for line in output.splitlines(): - if line.startswith("package:"): - pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") - thisinfo['id'] = re.match(pat, line).group(1) - pat = re.compile(".*versionCode='([0-9]*)'.*") - thisinfo['versioncode'] = int(re.match(pat, line).group(1)) - pat = re.compile(".*versionName='([^']*)'.*") - thisinfo['version'] = re.match(pat, line).group(1) - if line.startswith("application:"): - pat = re.compile(".*label='([^']*)'.*") - thisinfo['name'] = re.match(pat, line).group(1) - pat = re.compile(".*icon='([^']*)'.*") - thisinfo['iconsrc'] = re.match(pat, line).group(1) - if line.startswith("sdkVersion:"): - pat = re.compile(".*'([0-9]*)'.*") - thisinfo['sdkversion'] = re.match(pat, line).group(1) - if line.startswith("native-code:"): - pat = re.compile(".*'([^']*)'.*") - thisinfo['nativecode'] = re.match(pat, line).group(1) - if line.startswith("uses-permission:"): - pat = re.compile(".*'([^']*)'.*") - perm = re.match(pat, line).group(1) - if perm.startswith("android.permission."): - perm = perm[19:] - thisinfo['permissions'].append(perm) - if line.startswith("uses-feature:"): - pat = re.compile(".*'([^']*)'.*") - perm = re.match(pat, line).group(1) - #Filter out this, it's only added with the latest SDK tools and - #causes problems for lots of apps. - if (perm != "android.hardware.screen.portrait" and - perm != "android.hardware.screen.landscape"): - if perm.startswith("android.feature."): - perm = perm[16:] - thisinfo['features'].append(perm) + # Get all apps... + apps = common.read_metadata(verbose=options.verbose) - if not thisinfo.has_key('sdkversion'): - print " WARNING: no SDK version information found" - thisinfo['sdkversion'] = 0 - - # Calculate the md5 and sha256... - m = hashlib.md5() - sha = hashlib.sha256() - f = open(apkfile, 'rb') - while True: - t = f.read(1024) - if len(t) == 0: - break - m.update(t) - sha.update(t) - thisinfo['md5'] = m.hexdigest() - thisinfo['sha256'] = sha.hexdigest() - f.close() - - # Get the signature (or md5 of, to be precise)... - p = subprocess.Popen(['java', 'getsig', - os.path.join(os.getcwd(), apkfile)], - cwd=os.path.join(sys.path[0], 'getsig'), - stdout=subprocess.PIPE) - output = p.communicate()[0] - if options.verbose: - print output - if p.returncode != 0 or not output.startswith('Result:'): - print "ERROR: Failed to get apk signature" - sys.exit(1) - thisinfo['sig'] = output[7:].strip() - - # Extract the icon file... - apk = zipfile.ZipFile(apkfile, 'r') - thisinfo['icon'] = (thisinfo['id'] + '.' + - str(thisinfo['versioncode']) + '.png') - iconfilename = os.path.join(icon_dir, thisinfo['icon']) - try: - iconfile = open(iconfilename, 'wb') - iconfile.write(apk.read(thisinfo['iconsrc'])) - iconfile.close() - except: - print "WARNING: Error retrieving icon file" - warnings += 1 - apk.close() - - apks.append(thisinfo) - -# Some information from the apks needs to be applied up to the application -# level. When doing this, we use the info from the most recent version's apk. -for app in apps: - bestver = 0 - for apk in apks: - if apk['id'] == app['id']: - if apk['versioncode'] > bestver: - bestver = apk['versioncode'] - bestapk = apk - - if bestver == 0: - if app['Name'] is None: - app['Name'] = app['id'] - app['icon'] = '' - if app['Disabled'] is None: - print "WARNING: Application " + app['id'] + " has no packages" - else: - if app['Name'] is None: - app['Name'] = bestapk['name'] - app['icon'] = bestapk['icon'] - -# Generate warnings for apk's with no metadata (or create skeleton -# metadata files, if requested on the command line) -for apk in apks: - found = False + # Generate a list of categories... + categories = [] for app in apps: - if app['id'] == apk['id']: - found = True - break - if not found: - if options.createmeta: - f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w') - f.write("License:Unknown\n") - f.write("Web Site:\n") - f.write("Source Code:\n") - f.write("Issue Tracker:\n") - f.write("Summary:" + apk['name'] + "\n") - f.write("Description:\n") - f.write(apk['name'] + "\n") - f.write(".\n") - f.close() - print "Generated skeleton metadata for " + apk['id'] - else: - print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata" - print " " + apk['name'] + " - " + apk['version'] + if app['Category'] not in categories: + categories.append(app['Category']) -#Sort the app list by name, then the web site doesn't have to by default: -apps = sorted(apps, key=lambda app: app['Name'].upper()) + # Gather information about all the apk files in the repo directory... + apks = [] + for apkfile in glob.glob(os.path.join('repo','*.apk')): -# Create the index -doc = Document() - -def addElement(name, value, doc, parent): - el = doc.createElement(name) - el.appendChild(doc.createTextNode(value)) - parent.appendChild(el) - -root = doc.createElement("fdroid") -doc.appendChild(root) - -repoel = doc.createElement("repo") -repoel.setAttribute("name", repo_name) -repoel.setAttribute("icon", os.path.basename(repo_icon)) -repoel.setAttribute("url", repo_url) - -if repo_keyalias != None: - - # Generate a certificate fingerprint the same way keytool does it - # (but with slightly different formatting) - def cert_fingerprint(data): - digest = hashlib.sha1(data).digest() - ret = [] - for i in range(4): - ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5])) - return " ".join(ret) - - def extract_pubkey(): - p = subprocess.Popen(['keytool', '-exportcert', - '-alias', repo_keyalias, - '-keystore', keystore, - '-storepass', keystorepass], - stdout=subprocess.PIPE) - cert = p.communicate()[0] - if p.returncode != 0: - print "ERROR: Failed to get repo pubkey" + apkfilename = apkfile[5:] + if apkfilename.find(' ') != -1: + print "No spaces in APK filenames!" sys.exit(1) - global repo_pubkey_fingerprint - repo_pubkey_fingerprint = cert_fingerprint(cert) - return "".join("%02x" % ord(b) for b in cert) + srcfilename = apkfilename[:-4] + "_src.tar.gz" - repoel.setAttribute("pubkey", extract_pubkey()) + if not options.quiet: + print "Processing " + apkfilename + thisinfo = {} + thisinfo['apkname'] = apkfilename + if os.path.exists(os.path.join('repo', srcfilename)): + thisinfo['srcname'] = srcfilename + thisinfo['size'] = os.path.getsize(apkfile) + thisinfo['permissions'] = [] + thisinfo['features'] = [] + p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'), + 'dump', 'badging', apkfile], + stdout=subprocess.PIPE) + output = p.communicate()[0] + if options.verbose: + print output + if p.returncode != 0: + print "ERROR: Failed to get apk information" + sys.exit(1) + for line in output.splitlines(): + if line.startswith("package:"): + pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") + thisinfo['id'] = re.match(pat, line).group(1) + pat = re.compile(".*versionCode='([0-9]*)'.*") + thisinfo['versioncode'] = int(re.match(pat, line).group(1)) + pat = re.compile(".*versionName='([^']*)'.*") + thisinfo['version'] = re.match(pat, line).group(1) + if line.startswith("application:"): + pat = re.compile(".*label='([^']*)'.*") + thisinfo['name'] = re.match(pat, line).group(1) + pat = re.compile(".*icon='([^']*)'.*") + thisinfo['iconsrc'] = re.match(pat, line).group(1) + if line.startswith("sdkVersion:"): + pat = re.compile(".*'([0-9]*)'.*") + thisinfo['sdkversion'] = re.match(pat, line).group(1) + if line.startswith("native-code:"): + pat = re.compile(".*'([^']*)'.*") + thisinfo['nativecode'] = re.match(pat, line).group(1) + if line.startswith("uses-permission:"): + pat = re.compile(".*'([^']*)'.*") + perm = re.match(pat, line).group(1) + if perm.startswith("android.permission."): + perm = perm[19:] + thisinfo['permissions'].append(perm) + if line.startswith("uses-feature:"): + pat = re.compile(".*'([^']*)'.*") + perm = re.match(pat, line).group(1) + #Filter out this, it's only added with the latest SDK tools and + #causes problems for lots of apps. + if (perm != "android.hardware.screen.portrait" and + perm != "android.hardware.screen.landscape"): + if perm.startswith("android.feature."): + perm = perm[16:] + thisinfo['features'].append(perm) -addElement('description', repo_description, doc, repoel) -root.appendChild(repoel) + if not thisinfo.has_key('sdkversion'): + print " WARNING: no SDK version information found" + thisinfo['sdkversion'] = 0 -apps_inrepo = 0 -apps_disabled = 0 -apps_nopkg = 0 + # Calculate the md5 and sha256... + m = hashlib.md5() + sha = hashlib.sha256() + f = open(apkfile, 'rb') + while True: + t = f.read(1024) + if len(t) == 0: + break + m.update(t) + sha.update(t) + thisinfo['md5'] = m.hexdigest() + thisinfo['sha256'] = sha.hexdigest() + f.close() -for app in apps: + # Get the signature (or md5 of, to be precise)... + p = subprocess.Popen(['java', 'getsig', + os.path.join(os.getcwd(), apkfile)], + cwd=os.path.join(sys.path[0], 'getsig'), + stdout=subprocess.PIPE) + output = p.communicate()[0] + if options.verbose: + print output + if p.returncode != 0 or not output.startswith('Result:'): + print "ERROR: Failed to get apk signature" + sys.exit(1) + thisinfo['sig'] = output[7:].strip() - if app['Disabled'] is None: + # Extract the icon file... + apk = zipfile.ZipFile(apkfile, 'r') + thisinfo['icon'] = (thisinfo['id'] + '.' + + str(thisinfo['versioncode']) + '.png') + iconfilename = os.path.join(icon_dir, thisinfo['icon']) + try: + iconfile = open(iconfilename, 'wb') + iconfile.write(apk.read(thisinfo['iconsrc'])) + iconfile.close() + except: + print "WARNING: Error retrieving icon file" + warnings += 1 + apk.close() - # Get a list of the apks for this app... - gotcurrentver = False - apklist = [] + apks.append(thisinfo) + + # Some information from the apks needs to be applied up to the application + # level. When doing this, we use the info from the most recent version's apk. + for app in apps: + bestver = 0 for apk in apks: if apk['id'] == app['id']: - if str(apk['versioncode']) == app['Current Version Code']: - gotcurrentver = True - apklist.append(apk) + if apk['versioncode'] > bestver: + bestver = apk['versioncode'] + bestapk = apk - if len(apklist) == 0: - apps_nopkg += 1 + if bestver == 0: + if app['Name'] is None: + app['Name'] = app['id'] + app['icon'] = '' + if app['Disabled'] is None: + print "WARNING: Application " + app['id'] + " has no packages" else: - apps_inrepo += 1 - apel = doc.createElement("application") - apel.setAttribute("id", app['id']) - root.appendChild(apel) + if app['Name'] is None: + app['Name'] = bestapk['name'] + app['icon'] = bestapk['icon'] - addElement('id', app['id'], doc, apel) - addElement('name', app['Name'], doc, apel) - addElement('summary', app['Summary'], doc, apel) - addElement('icon', app['icon'], doc, apel) - addElement('description', - common.parse_description(app['Description']), doc, apel) - addElement('license', app['License'], doc, apel) - if 'Category' in app: - addElement('category', app['Category'], doc, apel) - addElement('web', app['Web Site'], doc, apel) - addElement('source', app['Source Code'], doc, apel) - addElement('tracker', app['Issue Tracker'], doc, apel) - if app['Donate'] != None: - addElement('donate', app['Donate'], doc, apel) - - # These elements actually refer to the current version (i.e. which - # one is recommended. They are historically mis-named, and need - # changing, but stay like this for now to support existing clients. - addElement('marketversion', app['Current Version'], doc, apel) - addElement('marketvercode', app['Current Version Code'], doc, apel) - - if not (app['AntiFeatures'] is None): - addElement('antifeatures', app['AntiFeatures'], doc, apel) - if app['Requires Root']: - addElement('requirements', 'root', doc, apel) - - # Sort the apk list into version order, just so the web site - # doesn't have to do any work by default... - apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True) - - # Check for duplicates - they will make the client unhappy... - for i in range(len(apklist) - 1): - if apklist[i]['versioncode'] == apklist[i+1]['versioncode']: - print "ERROR - duplicate versions" - print apklist[i]['apkname'] - print apklist[i+1]['apkname'] - sys.exit(1) - - for apk in apklist: - apkel = doc.createElement("package") - apel.appendChild(apkel) - addElement('version', apk['version'], doc, apkel) - addElement('versioncode', str(apk['versioncode']), doc, apkel) - addElement('apkname', apk['apkname'], doc, apkel) - if apk.has_key('srcname'): - addElement('srcname', apk['srcname'], doc, apkel) - for hash_type in ('sha256', 'md5'): - if not hash_type in apk: - continue - hashel = doc.createElement("hash") - hashel.setAttribute("type", hash_type) - hashel.appendChild(doc.createTextNode(apk[hash_type])) - apkel.appendChild(hashel) - addElement('sig', apk['sig'], doc, apkel) - addElement('size', str(apk['size']), doc, apkel) - addElement('sdkver', str(apk['sdkversion']), doc, apkel) - perms = "" - for p in apk['permissions']: - if len(perms) > 0: - perms += "," - perms += p - if len(perms) > 0: - addElement('permissions', perms, doc, apkel) - features = "" - for f in apk['features']: - if len(features) > 0: - features += "," - features += f - if len(features) > 0: - addElement('features', features, doc, apkel) - - if options.buildreport: - if len(app['builds']) == 0: - print ("WARNING: No builds defined for " + app['id'] + - " Source: " + app['Source Code']) - warnings += 1 + # Generate warnings for apk's with no metadata (or create skeleton + # metadata files, if requested on the command line) + for apk in apks: + found = False + for app in apps: + if app['id'] == apk['id']: + found = True + break + if not found: + if options.createmeta: + f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w') + f.write("License:Unknown\n") + f.write("Web Site:\n") + f.write("Source Code:\n") + f.write("Issue Tracker:\n") + f.write("Summary:" + apk['name'] + "\n") + f.write("Description:\n") + f.write(apk['name'] + "\n") + f.write(".\n") + f.close() + print "Generated skeleton metadata for " + apk['id'] else: - if app['Current Version Code'] != '0': - gotbuild = False - for build in app['builds']: - if build['vercode'] == app['Current Version Code']: - gotbuild = True - if not gotbuild: - print ("WARNING: No build data for current version of " - + app['id'] + " (" + app['Current Version'] - + ") " + app['Source Code']) - warnings += 1 + print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata" + print " " + apk['name'] + " - " + apk['version'] - # If we don't have the current version, check if there is a build - # with a commit ID starting with '!' - this means we can't build it - # for some reason, and don't want hassling about it... - if not gotcurrentver and app['Current Version Code'] != '0': - for build in app['builds']: - if build['vercode'] == app['Current Version Code']: - gotcurrentver = True + #Sort the app list by name, then the web site doesn't have to by default: + apps = sorted(apps, key=lambda app: app['Name'].upper()) - # Output a message of harassment if we don't have the current version: - if not gotcurrentver and app['Current Version Code'] != '0': - addr = app['Source Code'] - print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name'] - print " (" + app['id'] + ") " + addr - warnings += 1 - if options.verbose: - # A bit of extra debug info, basically for diagnosing - # app developer mistakes: - print " Current vercode:" + app['Current Version Code'] - print " Got:" - for apk in apks: - if apk['id'] == app['id']: - print " " + str(apk['versioncode']) + " - " + apk['version'] - if options.interactive: - print "Build data out of date for " + app['id'] - while True: - answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower() - if answer == 'i': - break - elif answer == 'e': - subprocess.call([options.editor, - os.path.join('metadata', - app['id'] + '.txt')]) - break - elif answer == 'q': - sys.exit(0) - else: - apps_disabled += 1 + # Create the index + doc = Document() -of = open(os.path.join('repo','index.xml'), 'wb') -if options.pretty: - output = doc.toprettyxml() -else: - output = doc.toxml() -of.write(output) -of.close() + def addElement(name, value, doc, parent): + el = doc.createElement(name) + el.appendChild(doc.createTextNode(value)) + parent.appendChild(el) -if repo_keyalias != None: + root = doc.createElement("fdroid") + doc.appendChild(root) - if not options.quiet: - print "Creating signed index." - print "Key fingerprint:", repo_pubkey_fingerprint - - #Create a jar of the index... - p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'], - cwd='repo', stdout=subprocess.PIPE) - output = p.communicate()[0] - if options.verbose: - print output - if p.returncode != 0: - print "ERROR: Failed to create jar file" - sys.exit(1) + repoel = doc.createElement("repo") + repoel.setAttribute("name", repo_name) + repoel.setAttribute("icon", os.path.basename(repo_icon)) + repoel.setAttribute("url", repo_url) - # Sign the index... - p = subprocess.Popen(['jarsigner', '-keystore', keystore, - '-storepass', keystorepass, '-keypass', keypass, - os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE) - output = p.communicate()[0] - if p.returncode != 0: - print "Failed to sign index" - print output - sys.exit(1) - if options.verbose: - print output + if repo_keyalias != None: -# Copy the repo icon into the repo directory... -iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon)) -shutil.copyfile(repo_icon, iconfilename) + # Generate a certificate fingerprint the same way keytool does it + # (but with slightly different formatting) + def cert_fingerprint(data): + digest = hashlib.sha1(data).digest() + ret = [] + for i in range(4): + ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5])) + return " ".join(ret) -# Write a category list in the repo to allow quick access... -catdata = '' -for cat in categories: - catdata += cat + '\n' -f = open('repo/categories.txt', 'w') -f.write(catdata) -f.close() + def extract_pubkey(): + p = subprocess.Popen(['keytool', '-exportcert', + '-alias', repo_keyalias, + '-keystore', keystore, + '-storepass', keystorepass], + stdout=subprocess.PIPE) + cert = p.communicate()[0] + if p.returncode != 0: + print "ERROR: Failed to get repo pubkey" + sys.exit(1) + global repo_pubkey_fingerprint + repo_pubkey_fingerprint = cert_fingerprint(cert) + return "".join("%02x" % ord(b) for b in cert) -# Update known apks info... -knownapks = common.KnownApks() -for apk in apks: - knownapks.recordapk(apk['apkname'], apk['id']) -knownapks.writeifchanged() + repoel.setAttribute("pubkey", extract_pubkey()) + + addElement('description', repo_description, doc, repoel) + root.appendChild(repoel) + + apps_inrepo = 0 + apps_disabled = 0 + apps_nopkg = 0 -# Generate latest apps data for widget -data = '' -for line in file(os.path.join('stats', 'latestapps.txt')): - appid = line.rstrip() - data += appid + "\t" for app in apps: - if app['id'] == appid: - data += app['Name'] + "\t" - data += app['icon'] + "\t" - data += app['License'] + "\n" - break -f = open('repo/latestapps.dat', 'w') -f.write(data) -f.close() + + if app['Disabled'] is None: + + # Get a list of the apks for this app... + gotcurrentver = False + apklist = [] + for apk in apks: + if apk['id'] == app['id']: + if str(apk['versioncode']) == app['Current Version Code']: + gotcurrentver = True + apklist.append(apk) + + if len(apklist) == 0: + apps_nopkg += 1 + else: + apps_inrepo += 1 + apel = doc.createElement("application") + apel.setAttribute("id", app['id']) + root.appendChild(apel) + + addElement('id', app['id'], doc, apel) + addElement('name', app['Name'], doc, apel) + addElement('summary', app['Summary'], doc, apel) + addElement('icon', app['icon'], doc, apel) + addElement('description', + common.parse_description(app['Description']), doc, apel) + addElement('license', app['License'], doc, apel) + if 'Category' in app: + addElement('category', app['Category'], doc, apel) + addElement('web', app['Web Site'], doc, apel) + addElement('source', app['Source Code'], doc, apel) + addElement('tracker', app['Issue Tracker'], doc, apel) + if app['Donate'] != None: + addElement('donate', app['Donate'], doc, apel) + + # These elements actually refer to the current version (i.e. which + # one is recommended. They are historically mis-named, and need + # changing, but stay like this for now to support existing clients. + addElement('marketversion', app['Current Version'], doc, apel) + addElement('marketvercode', app['Current Version Code'], doc, apel) + + if not (app['AntiFeatures'] is None): + addElement('antifeatures', app['AntiFeatures'], doc, apel) + if app['Requires Root']: + addElement('requirements', 'root', doc, apel) + + # Sort the apk list into version order, just so the web site + # doesn't have to do any work by default... + apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True) + + # Check for duplicates - they will make the client unhappy... + for i in range(len(apklist) - 1): + if apklist[i]['versioncode'] == apklist[i+1]['versioncode']: + print "ERROR - duplicate versions" + print apklist[i]['apkname'] + print apklist[i+1]['apkname'] + sys.exit(1) + + for apk in apklist: + apkel = doc.createElement("package") + apel.appendChild(apkel) + addElement('version', apk['version'], doc, apkel) + addElement('versioncode', str(apk['versioncode']), doc, apkel) + addElement('apkname', apk['apkname'], doc, apkel) + if apk.has_key('srcname'): + addElement('srcname', apk['srcname'], doc, apkel) + for hash_type in ('sha256', 'md5'): + if not hash_type in apk: + continue + hashel = doc.createElement("hash") + hashel.setAttribute("type", hash_type) + hashel.appendChild(doc.createTextNode(apk[hash_type])) + apkel.appendChild(hashel) + addElement('sig', apk['sig'], doc, apkel) + addElement('size', str(apk['size']), doc, apkel) + addElement('sdkver', str(apk['sdkversion']), doc, apkel) + perms = "" + for p in apk['permissions']: + if len(perms) > 0: + perms += "," + perms += p + if len(perms) > 0: + addElement('permissions', perms, doc, apkel) + features = "" + for f in apk['features']: + if len(features) > 0: + features += "," + features += f + if len(features) > 0: + addElement('features', features, doc, apkel) + + if options.buildreport: + if len(app['builds']) == 0: + print ("WARNING: No builds defined for " + app['id'] + + " Source: " + app['Source Code']) + warnings += 1 + else: + if app['Current Version Code'] != '0': + gotbuild = False + for build in app['builds']: + if build['vercode'] == app['Current Version Code']: + gotbuild = True + if not gotbuild: + print ("WARNING: No build data for current version of " + + app['id'] + " (" + app['Current Version'] + + ") " + app['Source Code']) + warnings += 1 + + # If we don't have the current version, check if there is a build + # with a commit ID starting with '!' - this means we can't build it + # for some reason, and don't want hassling about it... + if not gotcurrentver and app['Current Version Code'] != '0': + for build in app['builds']: + if build['vercode'] == app['Current Version Code']: + gotcurrentver = True + + # Output a message of harassment if we don't have the current version: + if not gotcurrentver and app['Current Version Code'] != '0': + addr = app['Source Code'] + print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name'] + print " (" + app['id'] + ") " + addr + warnings += 1 + if options.verbose: + # A bit of extra debug info, basically for diagnosing + # app developer mistakes: + print " Current vercode:" + app['Current Version Code'] + print " Got:" + for apk in apks: + if apk['id'] == app['id']: + print " " + str(apk['versioncode']) + " - " + apk['version'] + if options.interactive: + print "Build data out of date for " + app['id'] + while True: + answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower() + if answer == 'i': + break + elif answer == 'e': + subprocess.call([options.editor, + os.path.join('metadata', + app['id'] + '.txt')]) + break + elif answer == 'q': + sys.exit(0) + else: + apps_disabled += 1 + + of = open(os.path.join('repo','index.xml'), 'wb') + if options.pretty: + output = doc.toprettyxml() + else: + output = doc.toxml() + of.write(output) + of.close() + + if repo_keyalias != None: + + if not options.quiet: + print "Creating signed index." + print "Key fingerprint:", repo_pubkey_fingerprint + + #Create a jar of the index... + p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'], + cwd='repo', stdout=subprocess.PIPE) + output = p.communicate()[0] + if options.verbose: + print output + if p.returncode != 0: + print "ERROR: Failed to create jar file" + sys.exit(1) + + # Sign the index... + p = subprocess.Popen(['jarsigner', '-keystore', keystore, + '-storepass', keystorepass, '-keypass', keypass, + os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE) + output = p.communicate()[0] + if p.returncode != 0: + print "Failed to sign index" + print output + sys.exit(1) + if options.verbose: + print output + + # Copy the repo icon into the repo directory... + iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon)) + shutil.copyfile(repo_icon, iconfilename) + + # Write a category list in the repo to allow quick access... + catdata = '' + for cat in categories: + catdata += cat + '\n' + f = open('repo/categories.txt', 'w') + f.write(catdata) + f.close() + + # Update known apks info... + knownapks = common.KnownApks() + for apk in apks: + knownapks.recordapk(apk['apkname'], apk['id']) + knownapks.writeifchanged() + + # Generate latest apps data for widget + data = '' + for line in file(os.path.join('stats', 'latestapps.txt')): + appid = line.rstrip() + data += appid + "\t" + for app in apps: + if app['id'] == appid: + data += app['Name'] + "\t" + data += app['icon'] + "\t" + data += app['License'] + "\n" + break + f = open('repo/latestapps.dat', 'w') + f.write(data) + f.close() -print "Finished." -print str(apps_inrepo) + " apps in repo" -print str(apps_disabled) + " disabled" -print str(apps_nopkg) + " with no packages" -print str(warnings) + " warnings" + print "Finished." + print str(apps_inrepo) + " apps in repo" + print str(apps_disabled) + " disabled" + print str(apps_nopkg) + " with no packages" + print str(warnings) + " warnings" + +if __name__ == "__main__": + main() diff --git a/updatestats.py b/updatestats.py index 892c20bc..a6f98169 100755 --- a/updatestats.py +++ b/updatestats.py @@ -30,121 +30,125 @@ import HTMLParser import paramiko import common -#Read configuration... -execfile('config.py') +def main(): + + # Read configuration... + execfile('config.py') + + # Parse command line... + parser = OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + parser.add_option("-d", "--download", action="store_true", default=False, + help="Download logs we don't have") + (options, args) = parser.parse_args() -# Parse command line... -parser = OptionParser() -parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") -parser.add_option("-d", "--download", action="store_true", default=False, - help="Download logs we don't have") -(options, args) = parser.parse_args() + statsdir = 'stats' + logsdir = os.path.join(statsdir, 'logs') + logsarchivedir = os.path.join(logsdir, 'archive') + datadir = os.path.join(statsdir, 'data') + if not os.path.exists(statsdir): + os.mkdir(statsdir) + if not os.path.exists(logsdir): + os.mkdir(logsdir) + if not os.path.exists(datadir): + os.mkdir(datadir) + if options.download: + # Get any access logs we don't have... + ssh = None + ftp = None + try: + print 'Retrieving logs' + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + ssh.connect('f-droid.org', username='fdroid', timeout=10, + key_filename=webserver_keyfile) + ftp = ssh.open_sftp() + ftp.get_channel().settimeout(15) + print "...connected" -statsdir = 'stats' -logsdir = os.path.join(statsdir, 'logs') -logsarchivedir = os.path.join(logsdir, 'archive') -datadir = os.path.join(statsdir, 'data') -if not os.path.exists(statsdir): - os.mkdir(statsdir) -if not os.path.exists(logsdir): - os.mkdir(logsdir) -if not os.path.exists(datadir): - os.mkdir(datadir) + ftp.chdir('logs') + files = ftp.listdir() + for f in files: + if f.startswith('access-') and f.endswith('.log'): -if options.download: - # Get any access logs we don't have... - ssh = None - ftp = None - try: - print 'Retrieving logs' - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - ssh.connect('f-droid.org', username='fdroid', timeout=10, - key_filename=webserver_keyfile) - ftp = ssh.open_sftp() - ftp.get_channel().settimeout(15) - print "...connected" - - ftp.chdir('logs') - files = ftp.listdir() - for f in files: - if f.startswith('access-') and f.endswith('.log'): - - destpath = os.path.join(logsdir, f) - archivepath = os.path.join(logsarchivedir, f + '.gz') - if os.path.exists(archivepath): - if os.path.exists(destpath): - # Just in case we have it archived but failed to remove - # the original... - os.remove(destpath) - else: - destsize = ftp.stat(f).st_size - if (not os.path.exists(destpath) or - os.path.getsize(destpath) != destsize): - print "...retrieving " + f - ftp.get(f, destpath) - except Exception as e: - traceback.print_exc() - sys.exit(1) - finally: - #Disconnect - if ftp != None: - ftp.close() - if ssh != None: - ssh.close() - -# Process logs -logexpr = '(?P[.:0-9a-fA-F]+) - - \[(?P