kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-04 22:40:12 +01:00
Code Activity

Improve the stats bandwidth/diskspace/memory situation a bit

Browse Source
This commit is contained in:
Ciaran Gultnieks 2013-05-31 07:51:54 +01:00
parent 7707e0902e
commit 7878d959e7
1 changed files with 15 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
fdroidserver/stats.py
View File

@ -30,7 +30,7 @@ import HTMLParser
import paramiko import paramiko
import common import common
import socket import socket
import subprocess
def carbon_send(key, value): def carbon_send(key, value):
s = socket.socket() s = socket.socket()
@ -90,21 +90,14 @@ def main():
ftp.chdir('logs') ftp.chdir('logs')
files = ftp.listdir() files = ftp.listdir()
for f in files: for f in files:
if f.startswith('access-') and f.endswith('.log'): if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f) destpath = os.path.join(logsdir, f)
archivepath = os.path.join(logsarchivedir, f + '.gz') destsize = ftp.stat(f).st_size
if os.path.exists(archivepath): if (not os.path.exists(destpath) or
if os.path.exists(destpath): os.path.getsize(destpath) != destsize):
# Just in case we have it archived but failed to remove print "...retrieving " + f
# the original... ftp.get(f, destpath)
os.remove(destpath)
else:
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
print "...retrieving " + f
ftp.get(f, destpath)
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
sys.exit(1) sys.exit(1)
@ -116,14 +109,19 @@ def main():
ssh.close() ssh.close()
# Process logs # Process logs
if options.verbose:
print 'Processing logs...'
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"' logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search logsearch = re.compile(logexpr).search
apps = {} apps = {}
unknownapks = [] unknownapks = []
knownapks = common.KnownApks() knownapks = common.KnownApks()
for logfile in glob.glob(os.path.join(logsdir,'access-*.log')): for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
logdate = logfile[len(logsdir) + 1 + len('access-'):-4] if options.verbose:
matches = (logsearch(line) for line in file(logfile)) print '...' + logfile
logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
p = subprocess.Popen(["zcat", logfile], stdout = subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches: for match in matches:
if match and match.group('statuscode') == '200': if match and match.group('statuscode') == '200':
uri = match.group('uri') uri = match.group('uri')