diff --git a/examples/config.py b/examples/config.py index 1ed0de51..e46116db 100644 --- a/examples/config.py +++ b/examples/config.py @@ -173,10 +173,14 @@ The repository of older versions of applications from the main demo repository. # 'bar.info:/var/www/fdroid', # } -# Uncomment this option if you want to logs of builds and other processes to -# your repository server(s). Logs get published to all servers configured in -# 'serverwebroot'. The name scheme is: .../repo/$APPID_$VERCODE.log.gz -# Only logs from build-jobs running inside a buildserver VM are supported. +# When running fdroid processes on a remote server, it is possible to +# publish extra information about the status. Each fdroid sub-command +# can create repo/status/running.json when it starts, then a +# repo/status/.json when it completes. The builds logs +# and other processes will also get published, if they are running in +# a buildserver VM. The build logs name scheme is: +# .../repo/$APPID_$VERCODE.log.gz. These files are also pushed to all +# servers configured in 'serverwebroot'. # # deploy_process_logs = True diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 62f0187d..3697fec4 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -698,22 +698,29 @@ def setup_status_output(start_timestamp): 'commitId': get_head_commit_id(git_repo), 'isDirty': git_repo.is_dirty(), } + write_running_status_json(output) return output -def write_status_json(output, pretty=False): - """Write status out as JSON, and rsync it to the repo server""" +def write_running_status_json(output): + write_status_json(output, pretty=True, name='running') - subcommand = sys.argv[0].split()[1] + +def write_status_json(output, pretty=False, name=None): + """Write status out as JSON, and rsync it to the repo server""" status_dir = os.path.join('repo', 'status') if not os.path.exists(status_dir): os.mkdir(status_dir) - output['endTimestamp'] = int(datetime.now(timezone.utc).timestamp() * 1000) - with open(os.path.join(status_dir, subcommand + '.json'), 'w') as fp: + if not name: + output['endTimestamp'] = int(datetime.now(timezone.utc).timestamp() * 1000) + name = sys.argv[0].split()[1] # fdroid subcommand + path = os.path.join(status_dir, name + '.json') + with open(path, 'w') as fp: if pretty: json.dump(output, fp, sort_keys=True, cls=Encoder, indent=2) else: json.dump(output, fp, sort_keys=True, cls=Encoder, separators=(',', ':')) + rsync_status_file_to_repo(path, repo_subdir='status') def get_head_commit_id(git_repo): @@ -3350,11 +3357,6 @@ def deploy_build_log_with_rsync(appid, vercode, log_content): be decoded as 'utf-8') """ - # check if deploying logs is enabled in config - if not config.get('deploy_process_logs', False): - logging.debug(_('skip deploying full build logs: not enabled in config')) - return - if not log_content: logging.warning(_('skip deploying full build logs: log content is empty')) return @@ -3372,13 +3374,17 @@ def deploy_build_log_with_rsync(appid, vercode, log_content): f.write(bytes(log_content, 'utf-8')) else: f.write(log_content) + rsync_status_file_to_repo(log_gz_path) - # TODO: sign compressed log file, if a signing key is configured + +def rsync_status_file_to_repo(path, repo_subdir=None): + """Copy a build log or status JSON to the repo using rsync""" + + if not config.get('deploy_process_logs', False): + logging.debug(_('skip deploying full build logs: not enabled in config')) + return for webroot in config.get('serverwebroot', []): - dest_path = os.path.join(webroot, "repo") - if not dest_path.endswith('/'): - dest_path += '/' # make sure rsync knows this is a directory cmd = ['rsync', '--archive', '--delete-after', @@ -3389,15 +3395,21 @@ def deploy_build_log_with_rsync(appid, vercode, log_content): cmd += ['--quiet'] if 'identity_file' in config: cmd += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - cmd += [log_gz_path, dest_path] - # TODO: also deploy signature file if present + dest_path = os.path.join(webroot, "repo") + if repo_subdir is not None: + dest_path = os.path.join(dest_path, repo_subdir) + if not dest_path.endswith('/'): + dest_path += '/' # make sure rsync knows this is a directory + cmd += [path, dest_path] retcode = subprocess.call(cmd) if retcode: - logging.warning(_("failed deploying build logs to '{path}'").format(path=webroot)) + logging.error(_('process log deploy {path} to {dest} failed!') + .format(path=path, dest=webroot)) else: - logging.info(_("deployed build logs to '{path}'").format(path=webroot)) + logging.debug(_('deployed process log {path} to {dest}') + .format(path=path, dest=webroot)) def get_per_app_repos(): diff --git a/jenkins-test b/jenkins-test index d6ce9659..945d5d37 100755 --- a/jenkins-test +++ b/jenkins-test @@ -55,6 +55,7 @@ fi gpg --import $GNUPGHOME/secring.gpg echo "build_server_always = True" >> config.py +echo "deploy_process_logs = True" >> config.py echo "make_current_version_link = False" >> config.py echo "gpghome = '$GNUPGHOME'" >> config.py echo "gpgkey = 'CE71F7FB'" >> config.py @@ -66,6 +67,7 @@ test -d repo || mkdir repo test -d archive || mkdir archive # when everything is copied over to run on SIGN machine ../fdroid publish + ../fdroid gpgsign # when everything is copied over to run on BUILD machine, # which does not have a keyring, only a cached pubkey