mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-17 10:40:12 +02:00
More logging
This commit is contained in:
Daniel Martí
parent
e20fa9d7f4
commit
1b743e8b74
@ -462,8 +462,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
||||
jni_components = [c.strip() for c in jni_components.split(';')]
|
||||
ndkbuild = os.path.join(config['ndk_path'], "ndk-build")
|
||||
for d in jni_components:
|
||||
if options.verbose:
|
||||
print "Running ndk-build in " + root_dir + '/' + d
|
||||
logging.info("Building native code in '%s'" % d)
|
||||
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
|
||||
if os.path.exists(manifest):
|
||||
# Read and write the whole AM.xml to fix newlines and avoid
|
||||
@ -482,7 +481,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
||||
p = None
|
||||
# Build the release...
|
||||
if thisbuild['type'] == 'maven':
|
||||
print "Building Maven project..."
|
||||
logging.info("Building Maven project...")
|
||||
|
||||
if '@' in thisbuild['maven']:
|
||||
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
|
||||
@ -510,7 +509,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
||||
bindir = os.path.join(root_dir, 'target')
|
||||
|
||||
elif thisbuild['type'] == 'kivy':
|
||||
print "Building Kivy project..."
|
||||
logging.info("Building Kivy project...")
|
||||
|
||||
spec = os.path.join(root_dir, 'buildozer.spec')
|
||||
if not os.path.exists(spec):
|
||||
@ -570,7 +569,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
||||
p = FDroidPopen(cmd, cwd=distdir)
|
||||
|
||||
elif thisbuild['type'] == 'gradle':
|
||||
print "Building Gradle project..."
|
||||
logging.info("Building Gradle project...")
|
||||
if '@' in thisbuild['gradle']:
|
||||
flavours = thisbuild['gradle'].split('@')[0].split(',')
|
||||
gradle_dir = thisbuild['gradle'].split('@')[1]
|
||||
@ -601,7 +600,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
||||
p = FDroidPopen(commands, cwd=gradle_dir)
|
||||
|
||||
else:
|
||||
print "Building Ant project..."
|
||||
logging.info("Building Ant project...")
|
||||
cmd = ['ant']
|
||||
if 'antcommand' in thisbuild:
|
||||
cmd += [thisbuild['antcommand']]
|
||||
@ -613,7 +612,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
||||
|
||||
if p.returncode != 0:
|
||||
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout)
|
||||
print "Successfully built version " + thisbuild['version'] + ' of ' + app['id']
|
||||
logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id'])
|
||||
|
||||
# Find the apk name in the output...
|
||||
if 'bindir' in thisbuild:
|
||||
@ -659,7 +658,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
|
||||
|
||||
# By way of a sanity check, make sure the version and version
|
||||
# code in our new apk match what we expect...
|
||||
print "Checking " + src
|
||||
logging.info("Checking " + src)
|
||||
if not os.path.exists(src):
|
||||
raise BuildException("Unsigned apk is not at expected location of " + src)
|
||||
|
||||
@ -758,7 +757,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
|
||||
if 'disable' in thisbuild:
|
||||
return False
|
||||
|
||||
print "Building version " + thisbuild['version'] + ' of ' + app['id']
|
||||
logging.info("Building version " + thisbuild['version'] + ' of ' + app['id'])
|
||||
|
||||
if server:
|
||||
# When using server mode, still keep a local cache of the repo, by
|
||||
@ -826,12 +825,12 @@ def main():
|
||||
|
||||
log_dir = 'logs'
|
||||
if not os.path.isdir(log_dir):
|
||||
print "Creating log directory"
|
||||
logging.info("Creating log directory")
|
||||
os.makedirs(log_dir)
|
||||
|
||||
tmp_dir = 'tmp'
|
||||
if not os.path.isdir(tmp_dir):
|
||||
print "Creating temporary directory"
|
||||
logging.info("Creating temporary directory")
|
||||
os.makedirs(tmp_dir)
|
||||
|
||||
if options.test:
|
||||
@ -839,7 +838,7 @@ def main():
|
||||
else:
|
||||
output_dir = 'unsigned'
|
||||
if not os.path.isdir(output_dir):
|
||||
print "Creating output directory"
|
||||
logging.info("Creating output directory")
|
||||
os.makedirs(output_dir)
|
||||
|
||||
if config['archive_older'] != 0:
|
||||
@ -851,7 +850,7 @@ def main():
|
||||
|
||||
build_dir = 'build'
|
||||
if not os.path.isdir(build_dir):
|
||||
print "Creating build directory"
|
||||
logging.info("Creating build directory")
|
||||
os.makedirs(build_dir)
|
||||
srclib_dir = os.path.join(build_dir, 'srclib')
|
||||
extlib_dir = os.path.join(build_dir, 'extlib')
|
||||
@ -901,15 +900,13 @@ def main():
|
||||
build_dir = os.path.join('build', app['id'])
|
||||
|
||||
# Set up vcs interface and make sure we have the latest code...
|
||||
if options.verbose:
|
||||
print "Getting {0} vcs interface for {1}".format(
|
||||
app['Repo Type'], app['Repo'])
|
||||
logging.info("Getting {0} vcs interface for {1}".format(
|
||||
app['Repo Type'], app['Repo']))
|
||||
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
|
||||
|
||||
first = False
|
||||
|
||||
if options.verbose:
|
||||
print "Checking " + thisbuild['version']
|
||||
logging.info("Checking " + thisbuild['version'])
|
||||
if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir,
|
||||
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test,
|
||||
options.server, options.force, options.onserver):
|
||||
@ -919,19 +916,19 @@ def main():
|
||||
logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
|
||||
logfile.write(str(be))
|
||||
logfile.close()
|
||||
print "Could not build app %s due to BuildException: %s" % (app['id'], be)
|
||||
logging.info("Could not build app %s due to BuildException: %s" % (app['id'], be))
|
||||
if options.stop:
|
||||
sys.exit(1)
|
||||
failed_apps[app['id']] = be
|
||||
wikilog = be.get_wikitext()
|
||||
except VCSException as vcse:
|
||||
print "VCS error while building app %s: %s" % (app['id'], vcse)
|
||||
logging.info("VCS error while building app %s: %s" % (app['id'], vcse))
|
||||
if options.stop:
|
||||
sys.exit(1)
|
||||
failed_apps[app['id']] = vcse
|
||||
wikilog = str(vcse)
|
||||
except Exception as e:
|
||||
print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
|
||||
logging.info("Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc()))
|
||||
if options.stop:
|
||||
sys.exit(1)
|
||||
failed_apps[app['id']] = e
|
||||
@ -946,20 +943,20 @@ def main():
|
||||
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt
|
||||
newpage.save(txt, summary='Build log')
|
||||
except:
|
||||
print "Error while attempting to publish build log"
|
||||
logging.info("Error while attempting to publish build log")
|
||||
|
||||
for app in build_succeeded:
|
||||
print "success: %s" % (app['id'])
|
||||
logging.info("success: %s" % (app['id']))
|
||||
|
||||
if not options.verbose:
|
||||
for fa in failed_apps:
|
||||
print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
|
||||
logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
|
||||
|
||||
print "Finished."
|
||||
logging.info("Finished.")
|
||||
if len(build_succeeded) > 0:
|
||||
print str(len(build_succeeded)) + ' builds succeeded'
|
||||
logging.info(str(len(build_succeeded)) + ' builds succeeded')
|
||||
if len(failed_apps) > 0:
|
||||
print str(len(failed_apps)) + ' builds failed'
|
||||
logging.info(str(len(failed_apps)) + ' builds failed')
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
@ -28,6 +28,8 @@ from optparse import OptionParser
|
||||
import traceback
|
||||
import HTMLParser
|
||||
from distutils.version import LooseVersion
|
||||
import logging
|
||||
|
||||
import common, metadata
|
||||
from common import BuildException
|
||||
from common import VCSException
|
||||
@ -48,7 +50,7 @@ def check_http(app):
|
||||
|
||||
vercode = "99999999"
|
||||
if len(urlcode) > 0:
|
||||
print "...requesting {0}".format(urlcode)
|
||||
logging.info("...requesting {0}".format(urlcode))
|
||||
req = urllib2.Request(urlcode, None)
|
||||
resp = urllib2.urlopen(req, None, 20)
|
||||
page = resp.read()
|
||||
@ -61,7 +63,7 @@ def check_http(app):
|
||||
version = "??"
|
||||
if len(urlver) > 0:
|
||||
if urlver != '.':
|
||||
print "...requesting {0}".format(urlver)
|
||||
logging.info("...requesting {0}".format(urlver))
|
||||
req = urllib2.Request(urlver, None)
|
||||
resp = urllib2.urlopen(req, None, 20)
|
||||
page = resp.read()
|
||||
@ -114,16 +116,15 @@ def check_tags(app):
|
||||
hcode = "0"
|
||||
|
||||
for tag in vcs.gettags():
|
||||
if options.verbose:
|
||||
print "Check tag: '{0}'".format(tag)
|
||||
logging.info("Check tag: '{0}'".format(tag))
|
||||
vcs.gotorevision(tag)
|
||||
|
||||
# Only process tags where the manifest exists...
|
||||
paths = common.manifest_paths(build_dir, flavour)
|
||||
version, vercode, package = common.parse_androidmanifests(paths)
|
||||
if package and package == app['id'] and version and vercode:
|
||||
print "Manifest exists. Found version %s (%s)" % (
|
||||
version, vercode)
|
||||
logging.info("Manifest exists. Found version %s (%s)" % (
|
||||
version, vercode))
|
||||
if int(vercode) > int(hcode):
|
||||
htag = tag
|
||||
hcode = str(int(vercode))
|
||||
@ -201,7 +202,7 @@ def check_repomanifest(app, branch=None):
|
||||
|
||||
vercode = str(int(vercode))
|
||||
|
||||
print "Manifest exists. Found version %s (%s)" % (version, vercode)
|
||||
logging.info("Manifest exists. Found version %s (%s)" % (version, vercode))
|
||||
|
||||
return (version, vercode)
|
||||
|
||||
@ -307,38 +308,36 @@ def main():
|
||||
if options.gplay:
|
||||
for app in apps:
|
||||
version, reason = check_gplay(app)
|
||||
if version is None and options.verbose:
|
||||
if version is None:
|
||||
if reason == '404':
|
||||
print "%s is not in the Play Store" % common.getappname(app)
|
||||
logging.info("%s is not in the Play Store" % common.getappname(app))
|
||||
else:
|
||||
print "%s encountered a problem: %s" % (common.getappname(app), reason)
|
||||
logging.info("%s encountered a problem: %s" % (common.getappname(app), reason))
|
||||
if version is not None:
|
||||
stored = app['Current Version']
|
||||
if not stored:
|
||||
if options.verbose:
|
||||
print "%s has no Current Version but has version %s on the Play Store" % (
|
||||
common.getappname(app), version)
|
||||
logging.info("%s has no Current Version but has version %s on the Play Store" % (
|
||||
common.getappname(app), version))
|
||||
elif LooseVersion(stored) < LooseVersion(version):
|
||||
print "%s has version %s on the Play Store, which is bigger than %s" % (
|
||||
common.getappname(app), version, stored)
|
||||
elif options.verbose:
|
||||
logging.info("%s has version %s on the Play Store, which is bigger than %s" % (
|
||||
common.getappname(app), version, stored))
|
||||
else:
|
||||
if stored != version:
|
||||
print "%s has version %s on the Play Store, which differs from %s" % (
|
||||
common.getappname(app), version, stored)
|
||||
logging.info("%s has version %s on the Play Store, which differs from %s" % (
|
||||
common.getappname(app), version, stored))
|
||||
else:
|
||||
print "%s has the same version %s on the Play Store" % (
|
||||
common.getappname(app), version)
|
||||
logging.info("%s has the same version %s on the Play Store" % (
|
||||
common.getappname(app), version))
|
||||
return
|
||||
|
||||
|
||||
for app in apps:
|
||||
|
||||
if options.autoonly and app['Auto Update Mode'] == 'None':
|
||||
if options.verbose:
|
||||
print "Nothing to do for %s..." % app['id']
|
||||
logging.info("Nothing to do for %s..." % app['id'])
|
||||
continue
|
||||
|
||||
print "Processing " + app['id'] + '...'
|
||||
logging.info("Processing " + app['id'] + '...')
|
||||
|
||||
writeit = False
|
||||
logmsg = None
|
||||
@ -374,9 +373,9 @@ def main():
|
||||
|
||||
updating = False
|
||||
if not version:
|
||||
print "...%s" % msg
|
||||
logging.info("...%s" % msg)
|
||||
elif vercode == app['Current Version Code']:
|
||||
print "...up to date"
|
||||
logging.info("...up to date")
|
||||
else:
|
||||
app['Current Version'] = version
|
||||
app['Current Version Code'] = str(int(vercode))
|
||||
@ -413,12 +412,12 @@ def main():
|
||||
app['Current Version'] = cv
|
||||
writeit = True
|
||||
except Exception:
|
||||
print "ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc())
|
||||
logging.info("ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc()))
|
||||
|
||||
if updating:
|
||||
name = common.getappname(app)
|
||||
ver = common.getcvname(app)
|
||||
print '...updating to version %s' % ver
|
||||
logging.info('...updating to version %s' % ver)
|
||||
logmsg = 'Update CV of %s to %s' % (name, ver)
|
||||
|
||||
if options.auto:
|
||||
@ -447,7 +446,7 @@ def main():
|
||||
del newbuild['origlines']
|
||||
newbuild['vercode'] = app['Current Version Code']
|
||||
newbuild['version'] = app['Current Version'] + suffix
|
||||
print "...auto-generating build for " + newbuild['version']
|
||||
logging.info("...auto-generating build for " + newbuild['version'])
|
||||
commit = pattern.replace('%v', newbuild['version'])
|
||||
commit = commit.replace('%c', newbuild['vercode'])
|
||||
newbuild['commit'] = commit
|
||||
@ -457,23 +456,23 @@ def main():
|
||||
ver = common.getcvname(app)
|
||||
logmsg = "Update %s to %s" % (name, ver)
|
||||
else:
|
||||
print 'Invalid auto update mode "' + mode + '"'
|
||||
logging.info('Invalid auto update mode "' + mode + '"')
|
||||
|
||||
if writeit:
|
||||
metafile = os.path.join('metadata', app['id'] + '.txt')
|
||||
metadata.write_metadata(metafile, app)
|
||||
if options.commit and logmsg:
|
||||
print "Commiting update for " + metafile
|
||||
logging.info("Commiting update for " + metafile)
|
||||
gitcmd = ["git", "commit", "-m",
|
||||
logmsg]
|
||||
if 'auto_author' in config:
|
||||
gitcmd.extend(['--author', config['auto_author']])
|
||||
gitcmd.extend(["--", metafile])
|
||||
if subprocess.call(gitcmd) != 0:
|
||||
print "Git commit failed"
|
||||
logging.info("Git commit failed")
|
||||
sys.exit(1)
|
||||
|
||||
print "Finished."
|
||||
logging.info("Finished.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -24,6 +24,7 @@ import shutil
|
||||
import urllib
|
||||
from optparse import OptionParser
|
||||
from ConfigParser import ConfigParser
|
||||
import logging
|
||||
import common, metadata
|
||||
|
||||
# Get the repo type and address from the given web page. The page is scanned
|
||||
@ -106,13 +107,13 @@ def main():
|
||||
config = common.read_config(options)
|
||||
|
||||
if not options.url:
|
||||
print "Specify project url."
|
||||
logging.info("Specify project url.")
|
||||
sys.exit(1)
|
||||
url = options.url
|
||||
|
||||
tmp_dir = 'tmp'
|
||||
if not os.path.isdir(tmp_dir):
|
||||
print "Creating temporary directory"
|
||||
logging.info("Creating temporary directory")
|
||||
os.makedirs(tmp_dir)
|
||||
|
||||
# Get all apps...
|
||||
@ -133,7 +134,7 @@ def main():
|
||||
if url.endswith('/'):
|
||||
url = url[:-1]
|
||||
if url.endswith('.git'):
|
||||
print "A github URL should point to the project, not the git repo"
|
||||
logging.info("A github URL should point to the project, not the git repo")
|
||||
sys.exit(1)
|
||||
projecttype = 'github'
|
||||
repo = url + '.git'
|
||||
@ -154,7 +155,7 @@ def main():
|
||||
# Figure out the repo type and adddress...
|
||||
repotype, repo = getrepofrompage(sourcecode)
|
||||
if not repotype:
|
||||
print "Unable to determine vcs type. " + repo
|
||||
logging.info("Unable to determine vcs type. " + repo)
|
||||
sys.exit(1)
|
||||
elif url.startswith('http://code.google.com/p/'):
|
||||
if not url.endswith('/'):
|
||||
@ -168,29 +169,29 @@ def main():
|
||||
# Figure out the repo type and adddress...
|
||||
repotype, repo = getrepofrompage(sourcecode)
|
||||
if not repotype:
|
||||
print "Unable to determine vcs type. " + repo
|
||||
logging.info("Unable to determine vcs type. " + repo)
|
||||
sys.exit(1)
|
||||
|
||||
# Figure out the license...
|
||||
req = urllib.urlopen(url)
|
||||
if req.getcode() != 200:
|
||||
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
|
||||
logging.info('Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()))
|
||||
sys.exit(1)
|
||||
page = req.read()
|
||||
index = page.find('Code license')
|
||||
if index == -1:
|
||||
print "Couldn't find license data"
|
||||
logging.info("Couldn't find license data")
|
||||
sys.exit(1)
|
||||
ltext = page[index:]
|
||||
lprefix = 'rel="nofollow">'
|
||||
index = ltext.find(lprefix)
|
||||
if index == -1:
|
||||
print "Couldn't find license text"
|
||||
logging.info("Couldn't find license text")
|
||||
sys.exit(1)
|
||||
ltext = ltext[index + len(lprefix):]
|
||||
index = ltext.find('<')
|
||||
if index == -1:
|
||||
print "License text not formatted as expected"
|
||||
logging.info("License text not formatted as expected")
|
||||
sys.exit(1)
|
||||
ltext = ltext[:index]
|
||||
if ltext == 'GNU GPL v3':
|
||||
@ -208,17 +209,17 @@ def main():
|
||||
elif ltext == 'New BSD License':
|
||||
license = 'NewBSD'
|
||||
else:
|
||||
print "License " + ltext + " is not recognised"
|
||||
logging.info("License " + ltext + " is not recognised")
|
||||
sys.exit(1)
|
||||
|
||||
if not projecttype:
|
||||
print "Unable to determine the project type."
|
||||
print "The URL you supplied was not in one of the supported formats. Please consult"
|
||||
print "the manual for a list of supported formats, and supply one of those."
|
||||
logging.info("Unable to determine the project type.")
|
||||
logging.info("The URL you supplied was not in one of the supported formats. Please consult")
|
||||
logging.info("the manual for a list of supported formats, and supply one of those.")
|
||||
sys.exit(1)
|
||||
|
||||
# Get a copy of the source so we can extract some info...
|
||||
print 'Getting source from ' + repotype + ' repo at ' + repo
|
||||
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
|
||||
src_dir = os.path.join(tmp_dir, 'importer')
|
||||
if os.path.exists(src_dir):
|
||||
shutil.rmtree(src_dir)
|
||||
@ -235,12 +236,12 @@ def main():
|
||||
|
||||
version, vercode, package = common.parse_androidmanifests(paths)
|
||||
if not package:
|
||||
print "Couldn't find package ID"
|
||||
logging.info("Couldn't find package ID")
|
||||
sys.exit(1)
|
||||
if not version:
|
||||
print "WARNING: Couldn't find latest version name"
|
||||
logging.info("WARNING: Couldn't find latest version name")
|
||||
if not vercode:
|
||||
print "WARNING: Couldn't find latest version code"
|
||||
logging.info("WARNING: Couldn't find latest version code")
|
||||
else:
|
||||
spec = os.path.join(root_dir, 'buildozer.spec')
|
||||
if os.path.exists(spec):
|
||||
@ -252,13 +253,13 @@ def main():
|
||||
version = bconfig.get('app', 'version')
|
||||
vercode = None
|
||||
else:
|
||||
print "No android or kivy project could be found. Specify --subdir?"
|
||||
logging.info("No android or kivy project could be found. Specify --subdir?")
|
||||
sys.exit(1)
|
||||
|
||||
# Make sure it's actually new...
|
||||
for app in apps:
|
||||
if app['id'] == package:
|
||||
print "Package " + package + " already exists"
|
||||
logging.info("Package " + package + " already exists")
|
||||
sys.exit(1)
|
||||
|
||||
# Construct the metadata...
|
||||
@ -295,7 +296,7 @@ def main():
|
||||
|
||||
metafile = os.path.join('metadata', package + '.txt')
|
||||
metadata.write_metadata(metafile, app)
|
||||
print "Wrote " + metafile
|
||||
logging.info("Wrote " + metafile)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -26,6 +26,7 @@ import glob
|
||||
from optparse import OptionParser
|
||||
import paramiko
|
||||
import socket
|
||||
import logging
|
||||
|
||||
import common, metadata
|
||||
from common import FDroidPopen
|
||||
@ -57,7 +58,7 @@ def main():
|
||||
config = common.read_config(options)
|
||||
|
||||
if not config['update_stats']:
|
||||
print "Stats are disabled - check your configuration"
|
||||
logging.info("Stats are disabled - check your configuration")
|
||||
sys.exit(1)
|
||||
|
||||
# Get all metadata-defined apps...
|
||||
@ -78,14 +79,14 @@ def main():
|
||||
ssh = None
|
||||
ftp = None
|
||||
try:
|
||||
print 'Retrieving logs'
|
||||
logging.info('Retrieving logs')
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.load_system_host_keys()
|
||||
ssh.connect('f-droid.org', username='fdroid', timeout=10,
|
||||
key_filename=config['webserver_keyfile'])
|
||||
ftp = ssh.open_sftp()
|
||||
ftp.get_channel().settimeout(60)
|
||||
print "...connected"
|
||||
logging.info("...connected")
|
||||
|
||||
ftp.chdir('logs')
|
||||
files = ftp.listdir()
|
||||
@ -96,7 +97,7 @@ def main():
|
||||
destsize = ftp.stat(f).st_size
|
||||
if (not os.path.exists(destpath) or
|
||||
os.path.getsize(destpath) != destsize):
|
||||
print "...retrieving " + f
|
||||
logging.info("...retrieving " + f)
|
||||
ftp.get(f, destpath)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
@ -113,15 +114,13 @@ def main():
|
||||
|
||||
if not options.nologs:
|
||||
# Process logs
|
||||
if options.verbose:
|
||||
print 'Processing logs...'
|
||||
logging.info('Processing logs...')
|
||||
apps = {}
|
||||
appsVer = {}
|
||||
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
|
||||
logsearch = re.compile(logexpr).search
|
||||
for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
|
||||
if options.verbose:
|
||||
print '...' + logfile
|
||||
logging.info('...' + logfile)
|
||||
p = FDroidPopen(["zcat", logfile])
|
||||
matches = (logsearch(line) for line in p.stdout)
|
||||
for match in matches:
|
||||
@ -172,8 +171,7 @@ def main():
|
||||
f.close()
|
||||
|
||||
# Calculate and write stats for repo types...
|
||||
if options.verbose:
|
||||
print "Processing repo types..."
|
||||
logging.info("Processing repo types...")
|
||||
repotypes = {}
|
||||
for app in metaapps:
|
||||
if len(app['Repo Type']) == 0:
|
||||
@ -193,8 +191,7 @@ def main():
|
||||
f.close()
|
||||
|
||||
# Calculate and write stats for update check modes...
|
||||
if options.verbose:
|
||||
print "Processing update check modes..."
|
||||
logging.info("Processing update check modes...")
|
||||
ucms = {}
|
||||
for app in metaapps:
|
||||
checkmode = app['Update Check Mode'].split('/')[0]
|
||||
@ -207,8 +204,7 @@ def main():
|
||||
f.write(checkmode + ' ' + str(count) + '\n')
|
||||
f.close()
|
||||
|
||||
if options.verbose:
|
||||
print "Processing categories..."
|
||||
logging.info("Processing categories...")
|
||||
ctgs = {}
|
||||
for app in metaapps:
|
||||
if app['Categories'] is None:
|
||||
@ -224,8 +220,7 @@ def main():
|
||||
f.write(category + ' ' + str(count) + '\n')
|
||||
f.close()
|
||||
|
||||
if options.verbose:
|
||||
print "Processing antifeatures..."
|
||||
logging.info("Processing antifeatures...")
|
||||
afs = {}
|
||||
for app in metaapps:
|
||||
if app['AntiFeatures'] is None:
|
||||
@ -242,8 +237,7 @@ def main():
|
||||
f.close()
|
||||
|
||||
# Calculate and write stats for licenses...
|
||||
if options.verbose:
|
||||
print "Processing licenses..."
|
||||
logging.info("Processing licenses...")
|
||||
licenses = {}
|
||||
for app in metaapps:
|
||||
license = app['License']
|
||||
@ -257,8 +251,7 @@ def main():
|
||||
f.close()
|
||||
|
||||
# Write list of latest apps added to the repo...
|
||||
if options.verbose:
|
||||
print "Processing latest apps..."
|
||||
logging.info("Processing latest apps...")
|
||||
latest = knownapks.getlatest(10)
|
||||
f = open('stats/latestapps.txt', 'w')
|
||||
for app in latest:
|
||||
@ -266,11 +259,11 @@ def main():
|
||||
f.close()
|
||||
|
||||
if unknownapks:
|
||||
print '\nUnknown apks:'
|
||||
logging.info('\nUnknown apks:')
|
||||
for apk in unknownapks:
|
||||
print apk
|
||||
logging.info(apk)
|
||||
|
||||
print "Finished."
|
||||
logging.info("Finished.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user