rsync status JSON as first and last steps of command runs

examples/config.py

@@ -173,10 +173,14 @@ The repository of older versions of applications from the main demo repository.
 #     'bar.info:/var/www/fdroid',
 #     }
 
-# Uncomment this option if you want to logs of builds and other processes to
-# your repository server(s). Logs get published to all servers configured in
-# 'serverwebroot'. The name scheme is: .../repo/$APPID_$VERCODE.log.gz
-# Only logs from build-jobs running inside a buildserver VM are supported.
+# When running fdroid processes on a remote server, it is possible to
+# publish extra information about the status.  Each fdroid sub-command
+# can create repo/status/running.json when it starts, then a
+# repo/status/<sub-command>.json when it completes.  The builds logs
+# and other processes will also get published, if they are running in
+# a buildserver VM.  The build logs name scheme is:
+# .../repo/$APPID_$VERCODE.log.gz.  These files are also pushed to all
+# servers configured in 'serverwebroot'.
 #
 # deploy_process_logs = True
 

fdroidserver/common.py

@@ -698,22 +698,29 @@ def setup_status_output(start_timestamp):
             'commitId': get_head_commit_id(git_repo),
             'isDirty': git_repo.is_dirty(),
         }
+    write_running_status_json(output)
     return output
 
 
-def write_status_json(output, pretty=False):
-    """Write status out as JSON, and rsync it to the repo server"""
+def write_running_status_json(output):
+    write_status_json(output, pretty=True, name='running')
 
-    subcommand = sys.argv[0].split()[1]
+
+def write_status_json(output, pretty=False, name=None):
+    """Write status out as JSON, and rsync it to the repo server"""
     status_dir = os.path.join('repo', 'status')
     if not os.path.exists(status_dir):
         os.mkdir(status_dir)
-    output['endTimestamp'] =  int(datetime.now(timezone.utc).timestamp() * 1000)
-    with open(os.path.join(status_dir, subcommand + '.json'), 'w') as fp:
+    if not name:
+        output['endTimestamp'] = int(datetime.now(timezone.utc).timestamp() * 1000)
+        name = sys.argv[0].split()[1]  # fdroid subcommand
+    path = os.path.join(status_dir, name + '.json')
+    with open(path, 'w') as fp:
         if pretty:
             json.dump(output, fp, sort_keys=True, cls=Encoder, indent=2)
         else:
             json.dump(output, fp, sort_keys=True, cls=Encoder, separators=(',', ':'))
+    rsync_status_file_to_repo(path, repo_subdir='status')
 
 
 def get_head_commit_id(git_repo):
@@ -3350,11 +3357,6 @@ def deploy_build_log_with_rsync(appid, vercode, log_content):
                         be decoded as 'utf-8')
     """
 
-    # check if deploying logs is enabled in config
-    if not config.get('deploy_process_logs', False):
-        logging.debug(_('skip deploying full build logs: not enabled in config'))
-        return
-
     if not log_content:
         logging.warning(_('skip deploying full build logs: log content is empty'))
         return
@@ -3372,13 +3374,17 @@ def deploy_build_log_with_rsync(appid, vercode, log_content):
             f.write(bytes(log_content, 'utf-8'))
         else:
             f.write(log_content)
+    rsync_status_file_to_repo(log_gz_path)
 
-    # TODO: sign compressed log file, if a signing key is configured
+
+def rsync_status_file_to_repo(path, repo_subdir=None):
+    """Copy a build log or status JSON to the repo using rsync"""
+
+    if not config.get('deploy_process_logs', False):
+        logging.debug(_('skip deploying full build logs: not enabled in config'))
+        return
 
     for webroot in config.get('serverwebroot', []):
-        dest_path = os.path.join(webroot, "repo")
-        if not dest_path.endswith('/'):
-            dest_path += '/'  # make sure rsync knows this is a directory
         cmd = ['rsync',
                '--archive',
                '--delete-after',
@@ -3389,15 +3395,21 @@ def deploy_build_log_with_rsync(appid, vercode, log_content):
             cmd += ['--quiet']
         if 'identity_file' in config:
             cmd += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
-        cmd += [log_gz_path, dest_path]
 
-        # TODO: also deploy signature file if present
+        dest_path = os.path.join(webroot, "repo")
+        if repo_subdir is not None:
+            dest_path = os.path.join(dest_path, repo_subdir)
+        if not dest_path.endswith('/'):
+            dest_path += '/'  # make sure rsync knows this is a directory
+        cmd += [path, dest_path]
 
         retcode = subprocess.call(cmd)
         if retcode:
-            logging.warning(_("failed deploying build logs to '{path}'").format(path=webroot))
+            logging.error(_('process log deploy {path} to {dest} failed!')
+                          .format(path=path, dest=webroot))
         else:
-            logging.info(_("deployed build logs to '{path}'").format(path=webroot))
+            logging.debug(_('deployed process log {path} to {dest}')
+                          .format(path=path, dest=webroot))
 
 
 def get_per_app_repos():

jenkins-test

@@ -55,6 +55,7 @@ fi
 gpg --import $GNUPGHOME/secring.gpg
 
 echo "build_server_always = True" >> config.py
+echo "deploy_process_logs = True" >> config.py
 echo "make_current_version_link = False" >> config.py
 echo "gpghome = '$GNUPGHOME'" >> config.py
 echo "gpgkey = 'CE71F7FB'" >> config.py
@@ -66,6 +67,7 @@ test -d repo || mkdir repo
 test -d archive || mkdir archive
 # when everything is copied over to run on SIGN machine
 ../fdroid publish
+
 ../fdroid gpgsign
 # when everything is copied over to run on BUILD machine,
 # which does not have a keyring, only a cached pubkey