update: write status in JSON repo file, using new internal API

fdroidserver/common.py

@@ -20,6 +20,7 @@
 # common.py is imported by all modules, so do not import third-party
 # libraries here as they will become a requirement for all commands.
 
+import git
 import io
 import os
 import sys
@@ -47,7 +48,7 @@ except ImportError:
     import xml.etree.ElementTree as XMLElementTree  # nosec this is a fallback only
 
 from binascii import hexlify
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from distutils.version import LooseVersion
 from queue import Queue
 from zipfile import ZipFile
@@ -670,6 +671,51 @@ def get_build_dir(app):
     return os.path.join('build', app.id)
 
 
+class Encoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, set):
+            return sorted(obj)
+        return super().default(obj)
+
+
+def setup_status_output(start_timestamp):
+    """Create the common output dictionary for public status updates"""
+    output = {
+        'commandLine': sys.argv,
+        'startTimestamp': int(time.mktime(start_timestamp) * 1000),
+        'subcommand': sys.argv[0].split()[1],
+    }
+    if os.path.isdir('.git'):
+        git_repo = git.repo.Repo(os.getcwd())
+        output['fdroiddata'] = {
+            'commitId': get_head_commit_id(git_repo),
+            'isDirty': git_repo.is_dirty(),
+        }
+    fdroidserver_dir = os.path.dirname(sys.argv[0])
+    if os.path.isdir(os.path.join(fdroidserver_dir, '.git')):
+        git_repo = git.repo.Repo(fdroidserver_dir)
+        output['fdroidserver'] = {
+            'commitId': get_head_commit_id(git_repo),
+            'isDirty': git_repo.is_dirty(),
+        }
+    return output
+
+
+def write_status_json(output, pretty=False):
+    """Write status out as JSON, and rsync it to the repo server"""
+
+    subcommand = sys.argv[0].split()[1]
+    status_dir = os.path.join('repo', 'status')
+    if not os.path.exists(status_dir):
+        os.mkdir(status_dir)
+    output['endTimestamp'] =  int(datetime.now(timezone.utc).timestamp() * 1000)
+    with open(os.path.join(status_dir, subcommand + '.json'), 'w') as fp:
+        if pretty:
+            json.dump(output, fp, sort_keys=True, cls=Encoder, indent=2)
+        else:
+            json.dump(output, fp, sort_keys=True, cls=Encoder, separators=(',', ':'))
+
+
 def get_head_commit_id(git_repo):
     """Get git commit ID for HEAD as a str
 

fdroidserver/update.py

@@ -121,6 +121,57 @@ def disabled_algorithms_allowed():
     return options.allow_disabled_algorithms or config['allow_disabled_algorithms']
 
 
+def status_update_json(apps, sortedids, apks):
+    """Output a JSON file with metadata about this `fdroid update` run
+
+    :param apps: fully populated list of all applications
+    :param apks: all to be published apks
+
+    """
+
+    logging.debug(_('Outputting JSON'))
+    output = common.setup_status_output(start_timestamp)
+    output['antiFeatures'] = dict()
+    output['disabled'] = []
+    output['failedBuilds'] = dict()
+    output['noPackages'] = []
+
+    for appid in sortedids:
+        app = apps[appid]
+        for af in app.get('AntiFeatures', []):
+            antiFeatures = output['antiFeatures']  # JSON camelCase
+            if af not in antiFeatures:
+                antiFeatures[af] = dict()
+            if appid not in antiFeatures[af]:
+                antiFeatures[af]['apps'] = set()
+            antiFeatures[af]['apps'].add(appid)
+
+        apklist = []
+        for apk in apks:
+            if apk['packageName'] == appid:
+                apklist.append(apk)
+        builds = app.get('builds', [])
+        validapks = 0
+        for build in builds:
+            if not build.get('disable'):
+                builtit = False
+                for apk in apklist:
+                    if apk['versionCode'] == int(build.versionCode):
+                        builtit = True
+                        validapks += 1
+                        break
+                if not builtit:
+                    failedBuilds = output['failedBuilds']
+                    if appid not in failedBuilds:
+                        failedBuilds[appid] = []
+                    failedBuilds[appid].append(build.versionCode)
+        if validapks == 0:
+            output['noPackages'].append(appid)
+        if app.get('Disabled'):
+            output['disabled'].append(appid)
+    common.write_status_json(output, options.pretty)
+
+
 def update_wiki(apps, sortedids, apks):
     """Update the wiki
 
@@ -2200,6 +2251,7 @@ def main():
     # Update the wiki...
     if options.wiki:
         update_wiki(apps, sortedids, apks + archapks)
+    status_update_json(apps, sortedids, apks + archapks)
 
     logging.info(_("Finished"))
 

fdroidserver/verify.py

@@ -64,13 +64,6 @@ class Decoder(json.JSONDecoder):
         return set(values), end
 
 
-class Encoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, set):
-            return sorted(obj)
-        return super().default(obj)
-
-
 def write_json_report(url, remote_apk, unsigned_apk, compare_result):
     """write out the results of the verify run to JSON
 
@@ -118,7 +111,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result):
             data['packages'][packageName] = set()
         data['packages'][packageName].add(output)
         with open(jsonfile, 'w') as fp:
-            json.dump(data, fp, cls=Encoder, sort_keys=True)
+            json.dump(data, fp, cls=common.Encoder, sort_keys=True)
 
 
 def main():