mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-19 19:40:17 +02:00
271 lines
9.0 KiB
Python
271 lines
9.0 KiB
Python
#!/usr/bin/env python2
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# stats.py - part of the FDroid server tools
|
|
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import sys
|
|
import os
|
|
import re
|
|
import time
|
|
import traceback
|
|
import glob
|
|
from optparse import OptionParser
|
|
import paramiko
|
|
import socket
|
|
import logging
|
|
|
|
import common, metadata
|
|
from common import FDroidPopen
|
|
|
|
def carbon_send(key, value):
|
|
s = socket.socket()
|
|
s.connect((config['carbon_host'], config['carbon_port']))
|
|
msg = '%s %d %d\n' % (key, value, int(time.time()))
|
|
s.sendall(msg)
|
|
s.close()
|
|
|
|
options = None
|
|
config = None
|
|
|
|
def main():
|
|
|
|
global options, config
|
|
|
|
# Parse command line...
|
|
parser = OptionParser()
|
|
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
|
help="Spew out even more information than normal")
|
|
parser.add_option("-d", "--download", action="store_true", default=False,
|
|
help="Download logs we don't have")
|
|
parser.add_option("--nologs", action="store_true", default=False,
|
|
help="Don't do anything logs-related")
|
|
(options, args) = parser.parse_args()
|
|
|
|
config = common.read_config(options)
|
|
|
|
if not config['update_stats']:
|
|
logging.info("Stats are disabled - check your configuration")
|
|
sys.exit(1)
|
|
|
|
# Get all metadata-defined apps...
|
|
metaapps = metadata.read_metadata(options.verbose)
|
|
|
|
statsdir = 'stats'
|
|
logsdir = os.path.join(statsdir, 'logs')
|
|
datadir = os.path.join(statsdir, 'data')
|
|
if not os.path.exists(statsdir):
|
|
os.mkdir(statsdir)
|
|
if not os.path.exists(logsdir):
|
|
os.mkdir(logsdir)
|
|
if not os.path.exists(datadir):
|
|
os.mkdir(datadir)
|
|
|
|
if options.download:
|
|
# Get any access logs we don't have...
|
|
ssh = None
|
|
ftp = None
|
|
try:
|
|
logging.info('Retrieving logs')
|
|
ssh = paramiko.SSHClient()
|
|
ssh.load_system_host_keys()
|
|
ssh.connect('f-droid.org', username='fdroid', timeout=10,
|
|
key_filename=config['webserver_keyfile'])
|
|
ftp = ssh.open_sftp()
|
|
ftp.get_channel().settimeout(60)
|
|
logging.info("...connected")
|
|
|
|
ftp.chdir('logs')
|
|
files = ftp.listdir()
|
|
for f in files:
|
|
if f.startswith('access-') and f.endswith('.log.gz'):
|
|
|
|
destpath = os.path.join(logsdir, f)
|
|
destsize = ftp.stat(f).st_size
|
|
if (not os.path.exists(destpath) or
|
|
os.path.getsize(destpath) != destsize):
|
|
logging.info("...retrieving " + f)
|
|
ftp.get(f, destpath)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
finally:
|
|
#Disconnect
|
|
if ftp is not None:
|
|
ftp.close()
|
|
if ssh is not None:
|
|
ssh.close()
|
|
|
|
knownapks = common.KnownApks()
|
|
unknownapks = []
|
|
|
|
if not options.nologs:
|
|
# Process logs
|
|
logging.info('Processing logs...')
|
|
apps = {}
|
|
appsVer = {}
|
|
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
|
|
logsearch = re.compile(logexpr).search
|
|
for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
|
|
logging.info('...' + logfile)
|
|
p = FDroidPopen(["zcat", logfile])
|
|
matches = (logsearch(line) for line in p.stdout)
|
|
for match in matches:
|
|
if match and match.group('statuscode') == '200':
|
|
uri = match.group('uri')
|
|
if not uri.endswith('.apk'):
|
|
continue
|
|
_, apkname = os.path.split(uri)
|
|
app = knownapks.getapp(apkname)
|
|
if app:
|
|
appid, _ = app
|
|
if appid in apps:
|
|
apps[appid] += 1
|
|
else:
|
|
apps[appid] = 1
|
|
# Strip the '.apk' from apkname
|
|
appVer = apkname[:-4]
|
|
if appVer in appsVer:
|
|
appsVer[appVer] += 1
|
|
else:
|
|
appsVer[appVer] = 1
|
|
else:
|
|
if not apkname in unknownapks:
|
|
unknownapks.append(apkname)
|
|
|
|
# Calculate and write stats for total downloads...
|
|
lst = []
|
|
alldownloads = 0
|
|
for app, count in apps.iteritems():
|
|
lst.append(app + " " + str(count))
|
|
if config['stats_to_carbon']:
|
|
carbon_send('fdroid.download.' + app.replace('.', '_'), count)
|
|
alldownloads += count
|
|
lst.append("ALL " + str(alldownloads))
|
|
f = open('stats/total_downloads_app.txt', 'w')
|
|
f.write('# Total downloads by application, since October 2011\n')
|
|
for line in sorted(lst):
|
|
f.write(line + '\n')
|
|
f.close()
|
|
|
|
f = open('stats/total_downloads_app_version.txt', 'w')
|
|
f.write('# Total downloads by application and version, since October 2011\n')
|
|
lst = []
|
|
for appver, count in appsVer.iteritems():
|
|
lst.append(appver + " " + str(count))
|
|
for line in sorted(lst):
|
|
f.write(line + "\n")
|
|
f.close()
|
|
|
|
# Calculate and write stats for repo types...
|
|
logging.info("Processing repo types...")
|
|
repotypes = {}
|
|
for app in metaapps:
|
|
if len(app['Repo Type']) == 0:
|
|
rtype = 'none'
|
|
else:
|
|
if app['Repo Type'] == 'srclib':
|
|
rtype = common.getsrclibvcs(app['Repo'])
|
|
else:
|
|
rtype = app['Repo Type']
|
|
if rtype in repotypes:
|
|
repotypes[rtype] += 1;
|
|
else:
|
|
repotypes[rtype] = 1
|
|
f = open('stats/repotypes.txt', 'w')
|
|
for rtype, count in repotypes.iteritems():
|
|
f.write(rtype + ' ' + str(count) + '\n')
|
|
f.close()
|
|
|
|
# Calculate and write stats for update check modes...
|
|
logging.info("Processing update check modes...")
|
|
ucms = {}
|
|
for app in metaapps:
|
|
checkmode = app['Update Check Mode'].split('/')[0]
|
|
if checkmode in ucms:
|
|
ucms[checkmode] += 1;
|
|
else:
|
|
ucms[checkmode] = 1
|
|
f = open('stats/update_check_modes.txt', 'w')
|
|
for checkmode, count in ucms.iteritems():
|
|
f.write(checkmode + ' ' + str(count) + '\n')
|
|
f.close()
|
|
|
|
logging.info("Processing categories...")
|
|
ctgs = {}
|
|
for app in metaapps:
|
|
if app['Categories'] is None:
|
|
continue
|
|
categories = [c.strip() for c in app['Categories'].split(',')]
|
|
for category in categories:
|
|
if category in ctgs:
|
|
ctgs[category] += 1;
|
|
else:
|
|
ctgs[category] = 1
|
|
f = open('stats/categories.txt', 'w')
|
|
for category, count in ctgs.iteritems():
|
|
f.write(category + ' ' + str(count) + '\n')
|
|
f.close()
|
|
|
|
logging.info("Processing antifeatures...")
|
|
afs = {}
|
|
for app in metaapps:
|
|
if app['AntiFeatures'] is None:
|
|
continue
|
|
antifeatures = [a.strip() for a in app['AntiFeatures'].split(',')]
|
|
for antifeature in antifeatures:
|
|
if antifeature in afs:
|
|
afs[antifeature] += 1;
|
|
else:
|
|
afs[antifeature] = 1
|
|
f = open('stats/antifeatures.txt', 'w')
|
|
for antifeature, count in afs.iteritems():
|
|
f.write(antifeature + ' ' + str(count) + '\n')
|
|
f.close()
|
|
|
|
# Calculate and write stats for licenses...
|
|
logging.info("Processing licenses...")
|
|
licenses = {}
|
|
for app in metaapps:
|
|
license = app['License']
|
|
if license in licenses:
|
|
licenses[license] += 1;
|
|
else:
|
|
licenses[license] = 1
|
|
f = open('stats/licenses.txt', 'w')
|
|
for license, count in licenses.iteritems():
|
|
f.write(license + ' ' + str(count) + '\n')
|
|
f.close()
|
|
|
|
# Write list of latest apps added to the repo...
|
|
logging.info("Processing latest apps...")
|
|
latest = knownapks.getlatest(10)
|
|
f = open('stats/latestapps.txt', 'w')
|
|
for app in latest:
|
|
f.write(app + '\n')
|
|
f.close()
|
|
|
|
if unknownapks:
|
|
logging.info('\nUnknown apks:')
|
|
for apk in unknownapks:
|
|
logging.info(apk)
|
|
|
|
logging.info("Finished.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|