diff --git a/completion/bash-completion b/completion/bash-completion index dd32b30f..0f9c66d6 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -261,12 +261,6 @@ __complete_nightly() { __complete_options } -__complete_stats() { - opts="-v -q -d" - lopts="--verbose --quiet --download" - __complete_options -} - __complete_deploy() { opts="-i -v -q" lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir @@ -317,7 +311,6 @@ rewritemeta \ scanner \ signatures \ signindex \ -stats \ update \ verify \ " diff --git a/examples/config.yml b/examples/config.yml index a0943a8c..30e45270 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -289,27 +289,11 @@ # configured to allow push access (e.g. ssh key, username/password, etc) # binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git -# Only set this to true when running a repository where you want to generate -# stats, and only then on the master build servers, not a development -# machine. If you want to keep the "added" and "last updated" dates for each -# app and APK in your repo, then you should enable this. +# If you want to keep the "added" and "last updated" dates for each +# app and APK in your repo, enable this. The name comes from an old +# system for tracking statistics that is no longer included. # update_stats: true -# When used with stats, this is a list of IP addresses that are ignored for -# calculation purposes. -# stats_ignore: [] - -# Server stats logs are retrieved from. Required when update_stats is True. -# stats_server: example.com - -# User stats logs are retrieved from. Required when update_stats is True. -# stats_user: bob - -# Use the following to push stats to a Carbon instance: -# stats_to_carbon: false -# carbon_host: 0.0.0.0 -# carbon_port: 2003 - # Set this to true to always use a build server. This saves specifying the # --server option on dedicated secure build server hosts. # build_server_always: true diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 216e00c4..2c863d27 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -46,7 +46,6 @@ COMMANDS = OrderedDict([ ("rewritemeta", _("Rewrite all the metadata files")), ("lint", _("Warn about possible metadata errors")), ("scanner", _("Scan the source code of a package")), - ("stats", _("Update the stats of the repo")), ("signindex", _("Sign indexes created using update --nosign")), ("btlog", _("Update the binary transparency log for a URL")), ("signatures", _("Extract signatures from APKs")), diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ea3f219f..4595ebab 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -137,10 +137,6 @@ default_config = { 'current_version_name_source': 'Name', 'deploy_process_logs': False, 'update_stats': False, - 'stats_ignore': [], - 'stats_server': None, - 'stats_user': None, - 'stats_to_carbon': False, 'repo_maxage': 0, 'build_server_always': False, 'keystore': 'keystore.p12', diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py deleted file mode 100644 index a065ce18..00000000 --- a/fdroidserver/stats.py +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env python3 -# -# stats.py - part of the FDroid server tools -# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import sys -import os -import re -import time -import traceback -import glob -import json -from argparse import ArgumentParser -import paramiko -import socket -import logging -import subprocess -from collections import Counter - -from . import _ -from . import common -from . import metadata - - -def carbon_send(key, value): - s = socket.socket() - s.connect((config['carbon_host'], config['carbon_port'])) - msg = '%s %d %d\n' % (key, value, int(time.time())) - s.sendall(msg) - s.close() - - -options = None -config = None - - -def most_common_stable(counts): - pairs = [] - for s in counts: - pairs.append((s, counts[s])) - return sorted(pairs, key=lambda t: (-t[1], t[0])) - - -def main(): - - global options, config - - # Parse command line... - parser = ArgumentParser() - common.setup_global_opts(parser) - parser.add_argument("-d", "--download", action="store_true", default=False, - help=_("Download logs we don't have")) - parser.add_argument("--recalc", action="store_true", default=False, - help=_("Recalculate aggregate stats - use when changes " - "have been made that would invalidate old cached data.")) - parser.add_argument("--nologs", action="store_true", default=False, - help=_("Don't do anything logs-related")) - metadata.add_metadata_arguments(parser) - options = parser.parse_args() - metadata.warnings_action = options.W - - config = common.read_config(options) - - if not config['update_stats']: - logging.info("Stats are disabled - set \"update_stats = True\" in your config.yml") - sys.exit(1) - - # Get all metadata-defined apps... - allmetaapps = [app for app in metadata.read_metadata().values()] - metaapps = [app for app in allmetaapps if not app.Disabled] - - statsdir = 'stats' - logsdir = os.path.join(statsdir, 'logs') - datadir = os.path.join(statsdir, 'data') - if not os.path.exists(statsdir): - os.mkdir(statsdir) - if not os.path.exists(logsdir): - os.mkdir(logsdir) - if not os.path.exists(datadir): - os.mkdir(datadir) - - if options.download: - # Get any access logs we don't have... - ssh = None - ftp = None - try: - logging.info('Retrieving logs') - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - ssh.connect(config['stats_server'], username=config['stats_user'], - timeout=10, key_filename=config['webserver_keyfile']) - ftp = ssh.open_sftp() - ftp.get_channel().settimeout(60) - logging.info("...connected") - - ftp.chdir('logs') - files = ftp.listdir() - for f in files: - if f.startswith('access-') and f.endswith('.log.gz'): - - destpath = os.path.join(logsdir, f) - destsize = ftp.stat(f).st_size - if not os.path.exists(destpath) \ - or os.path.getsize(destpath) != destsize: - logging.debug("...retrieving " + f) - ftp.get(f, destpath) - except Exception: - traceback.print_exc() - sys.exit(1) - finally: - # Disconnect - if ftp is not None: - ftp.close() - if ssh is not None: - ssh.close() - - knownapks = common.KnownApks() - unknownapks = [] - - if not options.nologs: - # Process logs - logging.info('Processing logs...') - appscount = Counter() - appsvercount = Counter() - logexpr = r'(?P[.:0-9a-fA-F]+) - - \[(?P