diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7fffabb0..62f0187d 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -20,6 +20,7 @@ # common.py is imported by all modules, so do not import third-party # libraries here as they will become a requirement for all commands. +import git import io import os import sys @@ -47,7 +48,7 @@ except ImportError: import xml.etree.ElementTree as XMLElementTree # nosec this is a fallback only from binascii import hexlify -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from distutils.version import LooseVersion from queue import Queue from zipfile import ZipFile @@ -670,6 +671,51 @@ def get_build_dir(app): return os.path.join('build', app.id) +class Encoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return sorted(obj) + return super().default(obj) + + +def setup_status_output(start_timestamp): + """Create the common output dictionary for public status updates""" + output = { + 'commandLine': sys.argv, + 'startTimestamp': int(time.mktime(start_timestamp) * 1000), + 'subcommand': sys.argv[0].split()[1], + } + if os.path.isdir('.git'): + git_repo = git.repo.Repo(os.getcwd()) + output['fdroiddata'] = { + 'commitId': get_head_commit_id(git_repo), + 'isDirty': git_repo.is_dirty(), + } + fdroidserver_dir = os.path.dirname(sys.argv[0]) + if os.path.isdir(os.path.join(fdroidserver_dir, '.git')): + git_repo = git.repo.Repo(fdroidserver_dir) + output['fdroidserver'] = { + 'commitId': get_head_commit_id(git_repo), + 'isDirty': git_repo.is_dirty(), + } + return output + + +def write_status_json(output, pretty=False): + """Write status out as JSON, and rsync it to the repo server""" + + subcommand = sys.argv[0].split()[1] + status_dir = os.path.join('repo', 'status') + if not os.path.exists(status_dir): + os.mkdir(status_dir) + output['endTimestamp'] = int(datetime.now(timezone.utc).timestamp() * 1000) + with open(os.path.join(status_dir, subcommand + '.json'), 'w') as fp: + if pretty: + json.dump(output, fp, sort_keys=True, cls=Encoder, indent=2) + else: + json.dump(output, fp, sort_keys=True, cls=Encoder, separators=(',', ':')) + + def get_head_commit_id(git_repo): """Get git commit ID for HEAD as a str diff --git a/fdroidserver/update.py b/fdroidserver/update.py index ae4a4bfa..13947570 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -121,6 +121,57 @@ def disabled_algorithms_allowed(): return options.allow_disabled_algorithms or config['allow_disabled_algorithms'] +def status_update_json(apps, sortedids, apks): + """Output a JSON file with metadata about this `fdroid update` run + + :param apps: fully populated list of all applications + :param apks: all to be published apks + + """ + + logging.debug(_('Outputting JSON')) + output = common.setup_status_output(start_timestamp) + output['antiFeatures'] = dict() + output['disabled'] = [] + output['failedBuilds'] = dict() + output['noPackages'] = [] + + for appid in sortedids: + app = apps[appid] + for af in app.get('AntiFeatures', []): + antiFeatures = output['antiFeatures'] # JSON camelCase + if af not in antiFeatures: + antiFeatures[af] = dict() + if appid not in antiFeatures[af]: + antiFeatures[af]['apps'] = set() + antiFeatures[af]['apps'].add(appid) + + apklist = [] + for apk in apks: + if apk['packageName'] == appid: + apklist.append(apk) + builds = app.get('builds', []) + validapks = 0 + for build in builds: + if not build.get('disable'): + builtit = False + for apk in apklist: + if apk['versionCode'] == int(build.versionCode): + builtit = True + validapks += 1 + break + if not builtit: + failedBuilds = output['failedBuilds'] + if appid not in failedBuilds: + failedBuilds[appid] = [] + failedBuilds[appid].append(build.versionCode) + if validapks == 0: + output['noPackages'].append(appid) + if app.get('Disabled'): + output['disabled'].append(appid) + common.write_status_json(output, options.pretty) + + def update_wiki(apps, sortedids, apks): """Update the wiki @@ -2200,6 +2251,7 @@ def main(): # Update the wiki... if options.wiki: update_wiki(apps, sortedids, apks + archapks) + status_update_json(apps, sortedids, apks + archapks) logging.info(_("Finished")) diff --git a/fdroidserver/verify.py b/fdroidserver/verify.py index 8025b054..bd1433d2 100644 --- a/fdroidserver/verify.py +++ b/fdroidserver/verify.py @@ -64,13 +64,6 @@ class Decoder(json.JSONDecoder): return set(values), end -class Encoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, set): - return sorted(obj) - return super().default(obj) - - def write_json_report(url, remote_apk, unsigned_apk, compare_result): """write out the results of the verify run to JSON @@ -118,7 +111,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result): data['packages'][packageName] = set() data['packages'][packageName].add(output) with open(jsonfile, 'w') as fp: - json.dump(data, fp, cls=Encoder, sort_keys=True) + json.dump(data, fp, cls=common.Encoder, sort_keys=True) def main():