diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 14813fa1..073437f8 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -56,6 +56,14 @@ COMMANDS = OrderedDict([ ]) +# commands that are not advertised as public api, intended for the use-case +# of breaking down builds in buildbot into smaller steps +COMMANDS_INTERNAL = [ + "build_local_prepare", + "build_local_run", +] + + def print_help(available_plugins=None): print(_("usage: ") + _("fdroid [] [-h|--help|--version|]")) print("") @@ -136,7 +144,12 @@ def main(): sys.exit(0) command = sys.argv[1] - if command not in COMMANDS and command not in available_plugins: + command_not_found = ( + command not in COMMANDS + and command not in COMMANDS_INTERNAL + and command not in available_plugins + ) + if command_not_found: if command in ('-h', '--help'): print_help(available_plugins=available_plugins) sys.exit(0) @@ -186,7 +199,7 @@ def main(): sys.argv[0] += ' ' + command del sys.argv[1] - if command in COMMANDS.keys(): + if command in COMMANDS.keys() or command in COMMANDS_INTERNAL: # import is named import_subcommand internally b/c import is reserved by Python command = 'import_subcommand' if command == 'import' else command mod = __import__('fdroidserver.' + command, None, None, [command]) diff --git a/fdroidserver/build_local_prepare.py b/fdroidserver/build_local_prepare.py new file mode 100644 index 00000000..a495d0a8 --- /dev/null +++ b/fdroidserver/build_local_prepare.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +# build.py - part of the FDroid server tools +# Copyright (C) 2024, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import os +import pathlib +import logging +import argparse + +from fdroidserver import _ +import fdroidserver.common + + +def sudo_run(app, build): + # before doing anything, run the sudo commands to setup the VM + if build.sudo: + logging.info("Running 'sudo' commands in %s" % os.getcwd()) + + p = fdroidserver.common.FDroidPopen( + [ + 'sudo', + 'DEBIAN_FRONTEND=noninteractive', + 'bash', + '-e', + '-u', + '-o', + 'pipefail', + '-x', + '-c', + '; '.join(build.sudo), + ] + ) + if p.returncode != 0: + raise BuildException( + "Error running sudo command for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def sudo_lock_root(app, build): + p = fdroidserver.common.FDroidPopen(['sudo', 'passwd', '--lock', 'root']) + if p.returncode != 0: + raise BuildException( + "Error locking root account for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def sudo_uninstall(app, build): + p = fdroidserver.common.FDroidPopen( + ['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo'] + ) + if p.returncode != 0: + raise BuildException( + "Error removing sudo for %s:%s" % (app.id, build.versionName), p.output + ) + + +def log_tools_version(app, build, log_dir): + log_path = os.path.join( + log_dir, fdroidserver.common.get_toolsversion_logname(app, build) + ) + with open(log_path, 'w') as f: + f.write(fdroidserver.common.get_android_tools_version_log()) + + +def main(): + parser = argparse.ArgumentParser( + description=_( + "Download source code and initialize build environment " + "for one specific build" + ), + ) + parser.add_argument( + "--sudo-run", + action="store_true", + default=False, + help=_("run commands listed in sudo-metadata"), + ) + parser.add_argument( + "--sudo-uninstall", + action="store_true", + default=False, + help=_("uninstall sudo executing sudo-metadata"), + ) + parser.add_argument( + "--sudo-lock-root", + action="store_true", + default=False, + help=_("lock root user account"), + ) + parser.add_argument( + "APP_VERSION", + help=_("app id and version code tuple 'APPID:VERCODE'"), + ) + + # fdroid args/opts boilerplate + fdroidserver.common.setup_global_opts(parser) + options = fdroidserver.common.parse_args(parser) + config = fdroidserver.common.get_config() + + package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION) + app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) + + # prepare folders for git/vcs checkout + vcs, build_dir = fdroidserver.common.setup_vcs(app) + srclib_dir = pathlib.Path('./build/srclib') + extlib_dir = pathlib.Path('./build/extlib') + log_dir = pathlib.Path('./logs') + output_dir = pathlib.Path('./unsigned') + for d in (srclib_dir, extlib_dir, log_dir, output_dir): + d.mkdir(exist_ok=True, parents=True) + + # run sudo stuff + fdroidserver.common.set_FDroidPopen_env(build) + if options.sudo_run: + sudo_run(app, build) + if options.sudo_lock_root: + sudo_lock_root(app, build) + if options.sudo_uninstall: + sudo_uninstall(app, build) + + # TODO: in the past this was only logged when running as 'fdroid build + # --onserver' is this this output still valuable or can we remove it? + log_tools_version(app, build, log_dir) + + # do git/vcs checkout + fdroidserver.common.prepare_source( + vcs, app, build, build_dir, str(srclib_dir), str(extlib_dir) + ) + + # prepare for running cli commands + # NOTE: unclear if this is required here + # ndk_path = get_ndk_path(build) + # fdroidserver.common.set_FDroidPopen_env(build) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/build_local_run.py b/fdroidserver/build_local_run.py new file mode 100644 index 00000000..10f7caf5 --- /dev/null +++ b/fdroidserver/build_local_run.py @@ -0,0 +1,871 @@ +import os +import sys +import glob +import shutil +import gettext +import logging +import pathlib +import tarfile +import argparse +import traceback + +import fdroidserver.common +import fdroidserver.metadata +import fdroidserver.exception + +from fdroidserver import _ + + +def rlimit_check(apps_count): + """make sure enough open files are allowed to process everything""" + try: + import resource # not available on Windows + + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if apps_count > soft: + try: + soft = apps_count * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug( + _('Set open file limit to {integer}').format(integer=soft) + ) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) + except ImportError: + pass + + +def get_ndk_path(build): + ndk_path = build.ndk_path() + if build.ndk or (build.buildjni and build.buildjni != ['no']): + if not ndk_path: + logging.warning("Android NDK version '%s' could not be found!" % build.ndk) + logging.warning("Configured versions:") + for k, v in config['ndk_paths'].items(): + if k.endswith("_orig"): + continue + logging.warning(" %s: %s" % (k, v)) + if onserver: + fdroidserver.common.auto_install_ndk(build) + else: + raise fdroidserver.exception.FDroidException() + elif not os.path.isdir(ndk_path): + logging.critical("Android NDK '%s' is not a directory!" % ndk_path) + raise fdroidserver.exception.FDroidException() + + +def get_build_root_dir(app, build): + if build.subdir: + return os.path.join(fdroidserver.common.get_build_dir(app), build.subdir) + return fdroidserver.common.get_build_dir(app) + + +def transform_first_char(string, method): + """Use method() on the first character of string.""" + if len(string) == 0: + return string + if len(string) == 1: + return method(string) + return method(string[0]) + string[1:] + + +def init_build_subprocess(app, build, config): + # We need to clean via the build tool in case the binary dirs are + # different from the default ones + root_dir = get_build_root_dir(app, build) + + p = None + gradletasks = [] + flavours_cmd = None + + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Cleaning Maven project...") + cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']] + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + maven_dir = os.path.normpath(maven_dir) + else: + maven_dir = root_dir + + p = fdroidserver.common.FDroidPopen(cmd, cwd=maven_dir) + + elif bmethod == 'gradle': + + logging.info("Cleaning Gradle project...") + + if build.preassemble: + gradletasks += build.preassemble + + flavours = build.gradle + if flavours == ['yes']: + flavours = [] + + flavours_cmd = ''.join( + [transform_first_char(flav, str.upper) for flav in flavours] + ) + + gradletasks += ['assemble' + flavours_cmd + 'Release'] + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += ['clean'] + p = fdroidserver.common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Cleaning Ant project...") + p = fdroidserver.common.FDroidPopen(['ant', 'clean'], cwd=root_dir) + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error cleaning %s:%s" % (app.id, build.versionName), p.output + ) + + return p, gradletasks, flavours_cmd + + +def sanitize_build_dir(app): + build_dir = fdroidserver.common.get_build_dir(app) + for root, dirs, files in os.walk(build_dir): + + def del_dirs(dl): + for d in dl: + shutil.rmtree(os.path.join(root, d), ignore_errors=True) + + def del_files(fl): + for f in fl: + if f in files: + os.remove(os.path.join(root, f)) + + if any( + f in files + for f in [ + 'build.gradle', + 'build.gradle.kts', + 'settings.gradle', + 'settings.gradle.kts', + ] + ): + # Even when running clean, gradle stores task/artifact caches in + # .gradle/ as binary files. To avoid overcomplicating the scanner, + # manually delete them, just like `gradle clean` should have removed + # the build/* dirs. + del_dirs( + [ + os.path.join('build', 'android-profile'), + os.path.join('build', 'generated'), + os.path.join('build', 'intermediates'), + os.path.join('build', 'outputs'), + os.path.join('build', 'reports'), + os.path.join('build', 'tmp'), + os.path.join('buildSrc', 'build'), + '.gradle', + ] + ) + del_files(['gradlew', 'gradlew.bat']) + + if 'pom.xml' in files: + del_dirs(['target']) + + if any( + f in files for f in ['ant.properties', 'project.properties', 'build.xml'] + ): + del_dirs(['bin', 'gen']) + + if 'jni' in dirs: + del_dirs(['obj']) + + +def make_tarball(app, build, tmp_dir): + build_dir = fdroidserver.common.get_build_dir(app) + # Build the source tarball right before we build the release... + logging.info("Creating source tarball...") + tarname = fdroidserver.common.getsrcname(app, build) + tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") + + def tarexc(t): + return ( + None + if any(t.name.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) + else t + ) + + tarball.add(build_dir, tarname, filter=tarexc) + tarball.close() + + +def execute_build_commands(app, build): + build_dir = fdroidserver.common.get_build_dir(app) + srclibpaths = get_srclibpaths(build, build_dir) + if build.build: + logging.info("Running 'build' commands in %s" % root_dir) + cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build) + + # Substitute source library paths into commands... + for name, number, libpath in srclibpaths: + cmd = cmd.replace('$$' + name + '$$', os.path.join(os.getcwd(), libpath)) + + p = fdroidserver.common.FDroidPopen( + ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir + ) + + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error running build command for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def get_srclibpaths(build, srclib_dir): + srclibpaths = [] + if build.srclibs: + logging.info("Collecting source libraries") + for lib in build.srclibs: + srclibpaths.append( + fdroidserver.common.getsrclib( + lib, srclib_dir, preponly=onserver, refresh=refresh, build=build + ) + ) + return srclibpaths + + +def execute_buildjni_commands(app, build): + root_dir = get_build_root_dir(app, build) + ndk_path = get_ndk_path(build) + if build.buildjni and build.buildjni != ['no']: + logging.info("Building the native code") + jni_components = build.buildjni + + if jni_components == ['yes']: + jni_components = [''] + cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"] + for d in jni_components: + if d: + logging.info("Building native code in '%s'" % d) + else: + logging.info("Building native code in the main project") + manifest = os.path.join(root_dir, d, 'AndroidManifest.xml') + if os.path.exists(manifest): + # Read and write the whole AM.xml to fix newlines and avoid + # the ndk r8c or later 'wordlist' errors. The outcome of this + # under gnu/linux is the same as when using tools like + # dos2unix, but the native python way is faster and will + # work in non-unix systems. + manifest_text = open(manifest, 'U').read() + open(manifest, 'w').write(manifest_text) + # In case the AM.xml read was big, free the memory + del manifest_text + p = fdroidserver.common.FDroidPopen(cmd, cwd=os.path.join(root_dir, d)) + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "NDK build failed for %s:%s" % (app.id, build.versionName), p.output + ) + + +def execute_build___(app, build, config, gradletasks): + build_dir = fdroidserver.common.get_build_dir(app) + root_dir = get_build_root_dir(app, build) + + p = None + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Building Maven project...") + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + else: + maven_dir = root_dir + + mvncmd = [ + config['mvn3'], + '-Dandroid.sdk.path=' + config['sdk_path'], + '-Dmaven.jar.sign.skip=true', + '-Dmaven.test.skip=true', + '-Dandroid.sign.debug=false', + '-Dandroid.release=true', + 'package', + ] + if build.target: + target = build.target.split('-')[1] + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(root_dir, 'pom.xml'), + ) + if '@' in build.maven: + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(maven_dir, 'pom.xml'), + ) + + p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir) + + bindir = os.path.join(root_dir, 'target') + + elif bmethod == 'gradle': + logging.info("Building Gradle project...") + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += gradletasks + + p = fdroidserver.common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Building Ant project...") + cmd = ['ant'] + if build.antcommands: + cmd += build.antcommands + else: + cmd += ['release'] + p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir) + + bindir = os.path.join(root_dir, 'bin') + + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + + commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output, + ) + logging.info( + "Successfully built version {versionName} of {appid} from {commit_id}".format( + versionName=build.versionName, appid=app.id, commit_id=commit_id + ) + ) + + return p + + +def collect_build_output(app, build, p, flavours_cmd): + root_dir = get_build_root_dir(app, build) + + omethod = build.output_method() + if omethod == 'maven': + stdout_apk = '\n'.join( + [ + line + for line in p.output.splitlines() + if any(a in line for a in ('.apk', '.ap_', '.jar')) + ] + ) + m = re.match( + r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M + ) + if not m: + m = re.match( + r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", + stdout_apk, + re.S | re.M, + ) + if not m: + m = re.match( + r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + + bindir + + r'/([^/]+)\.ap[_k][,\]]', + stdout_apk, + re.S | re.M, + ) + + if not m: + m = re.match( + r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", + stdout_apk, + re.S | re.M, + ) + if not m: + raise fdroidserver.exception.BuildException('Failed to find output') + src = m.group(1) + src = os.path.join(bindir, src) + '.apk' + + elif omethod == 'gradle': + src = None + apk_dirs = [ + # gradle plugin >= 3.0 + os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'), + # gradle plugin < 3.0 and >= 0.11 + os.path.join(root_dir, 'build', 'outputs', 'apk'), + # really old path + os.path.join(root_dir, 'build', 'apk'), + ] + # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in + # a subdirectory corresponding to the flavour command used, but with different + # capitalization. + if flavours_cmd: + apk_dirs.append( + os.path.join( + root_dir, + 'build', + 'outputs', + 'apk', + transform_first_char(flavours_cmd, str.lower), + 'release', + ) + ) + for apks_dir in apk_dirs: + for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: + apks = glob.glob(os.path.join(apks_dir, apkglob)) + + if len(apks) > 1: + raise fdroidserver.exception.BuildException( + 'More than one resulting apks found in %s' % apks_dir, + '\n'.join(apks), + ) + if len(apks) == 1: + src = apks[0] + break + if src is not None: + break + + if src is None: + raise fdroidserver.exception.BuildException('Failed to find any output apks') + + elif omethod == 'ant': + stdout_apk = '\n'.join( + [line for line in p.output.splitlines() if '.apk' in line] + ) + src = re.match( + r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M + ).group(1) + src = os.path.join(bindir, src) + elif omethod == 'raw': + output_path = fdroidserver.common.replace_build_vars(build.output, build) + globpath = os.path.join(root_dir, output_path) + apks = glob.glob(globpath) + if len(apks) > 1: + raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) + if len(apks) < 1: + raise fdroidserver.exception.BuildException('No apks match %s' % globpath) + src = os.path.normpath(apks[0]) + + +def buildbuildbuild___(app, build, config, gradletasks): + root_dir = get_build_root_dir(app, build) + + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Building Maven project...") + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + else: + maven_dir = root_dir + + mvncmd = [ + config['mvn3'], + '-Dandroid.sdk.path=' + config['sdk_path'], + '-Dmaven.jar.sign.skip=true', + '-Dmaven.test.skip=true', + '-Dandroid.sign.debug=false', + '-Dandroid.release=true', + 'package', + ] + if build.target: + target = build.target.split('-')[1] + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(root_dir, 'pom.xml'), + ) + if '@' in build.maven: + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(maven_dir, 'pom.xml'), + ) + + p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir) + + bindir = os.path.join(root_dir, 'target') + + elif bmethod == 'gradle': + logging.info("Building Gradle project...") + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += gradletasks + + p = fdroidserver.common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Building Ant project...") + cmd = ['ant'] + if build.antcommands: + cmd += build.antcommands + else: + cmd += ['release'] + p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir) + + bindir = os.path.join(root_dir, 'bin') + + +def check_build_success(app, build, p): + build_dir = fdroidserver.common.get_build_dir(app) + + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + + commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output, + ) + logging.info( + "Successfully built version {versionName} of {appid} from {commit_id}".format( + versionName=build.versionName, appid=app.id, commit_id=commit_id + ) + ) + + +def collect_output_again(app, build, p, flavours_cmd): + root_dir = get_build_root_dir(app, build) + + omethod = build.output_method() + src = None + if omethod == 'maven': + stdout_apk = '\n'.join( + [ + line + for line in p.output.splitlines() + if any(a in line for a in ('.apk', '.ap_', '.jar')) + ] + ) + m = re.match( + r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M + ) + if not m: + m = re.match( + r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", + stdout_apk, + re.S | re.M, + ) + if not m: + m = re.match( + r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + + bindir + + r'/([^/]+)\.ap[_k][,\]]', + stdout_apk, + re.S | re.M, + ) + + if not m: + m = re.match( + r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", + stdout_apk, + re.S | re.M, + ) + if not m: + raise fdroidserver.exception.BuildException('Failed to find output') + src = m.group(1) + src = os.path.join(bindir, src) + '.apk' + + elif omethod == 'gradle': + src = None + apk_dirs = [ + # gradle plugin >= 3.0 + os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'), + # gradle plugin < 3.0 and >= 0.11 + os.path.join(root_dir, 'build', 'outputs', 'apk'), + # really old path + os.path.join(root_dir, 'build', 'apk'), + ] + # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in + # a subdirectory corresponding to the flavour command used, but with different + # capitalization. + if flavours_cmd: + apk_dirs.append( + os.path.join( + root_dir, + 'build', + 'outputs', + 'apk', + transform_first_char(flavours_cmd, str.lower), + 'release', + ) + ) + for apks_dir in apk_dirs: + for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: + apks = glob.glob(os.path.join(apks_dir, apkglob)) + + if len(apks) > 1: + raise fdroidserver.exception.BuildException( + 'More than one resulting apks found in %s' % apks_dir, + '\n'.join(apks), + ) + if len(apks) == 1: + src = apks[0] + break + if src is not None: + break + + if src is None: + raise fdroidserver.exception.BuildException('Failed to find any output apks') + + elif omethod == 'ant': + stdout_apk = '\n'.join( + [line for line in p.output.splitlines() if '.apk' in line] + ) + src = re.match( + r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M + ).group(1) + src = os.path.join(bindir, src) + elif omethod == 'raw': + output_path = fdroidserver.common.replace_build_vars(build.output, build) + globpath = os.path.join(root_dir, output_path) + apks = glob.glob(globpath) + if len(apks) > 1: + raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) + if len(apks) < 1: + raise fdroidserver.exception.BuildException('No apks match %s' % globpath) + src = os.path.normpath(apks[0]) + return src + + +def execute_postbuild(app, build, src): + build_dir = fdroidserver.common.get_build_dir(app) + root_dir = get_build_root_dir(app, build) + srclibpaths = get_srclibpaths(build, build_dir) + + if build.postbuild: + logging.info(f"Running 'postbuild' commands in {root_dir}") + cmd = fdroidserver.common.replace_config_vars("; ".join(build.postbuild), build) + + # Substitute source library paths into commands... + for name, number, libpath in srclibpaths: + cmd = cmd.replace(f"$${name}$$", str(Path.cwd() / libpath)) + + cmd = cmd.replace('$$OUT$$', str(Path(src).resolve())) + + p = fdroidserver.common.FDroidPopen( + ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir + ) + + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error running postbuild command for " f"{app.id}:{build.versionName}", + p.output, + ) + + +def get_metadata_from_apk(app, build, apkfile): + """Get the required metadata from the built APK. + + VersionName is allowed to be a blank string, i.e. '' + + Parameters + ---------- + app + The app metadata used to build the APK. + build + The build that resulted in the APK. + apkfile + The path of the APK file. + + Returns + ------- + versionCode + The versionCode from the APK or from the metadata is build.novcheck is + set. + versionName + The versionName from the APK or from the metadata is build.novcheck is + set. + + Raises + ------ + :exc:`~fdroidserver.exception.BuildException` + If native code should have been built but was not packaged, no version + information or no package ID could be found or there is a mismatch + between the package ID in the metadata and the one found in the APK. + """ + appid, versionCode, versionName = fdroidserver.common.get_apk_id(apkfile) + native_code = fdroidserver.common.get_native_code(apkfile) + + if build.buildjni and build.buildjni != ['no'] and not native_code: + raise fdroidserver.exception.BuildException("Native code should have been built but none was packaged") + if build.novcheck: + versionCode = build.versionCode + versionName = build.versionName + if not versionCode or versionName is None: + raise fdroidserver.exception.BuildException("Could not find version information in build in output") + if not appid: + raise fdroidserver.exception.BuildException("Could not find package ID in output") + if appid != app.id: + raise fdroidserver.exception.BuildException( + "Wrong package ID - build " + appid + " but expected " + app.id + ) + + return versionCode, versionName + + +def validate_build_artifacts(app, build, src): + # Make sure it's not debuggable... + if fdroidserver.common.is_debuggable_or_testOnly(src): + raise fdroidserver.exception.BuildException( + "%s: debuggable or testOnly set in AndroidManifest.xml" % src + ) + + # By way of a sanity check, make sure the version and version + # code in our new APK match what we expect... + logging.debug("Checking " + src) + if not os.path.exists(src): + raise fdroidserver.exception.BuildException("Unsigned APK is not at expected location of " + src) + + if fdroidserver.common.get_file_extension(src) == 'apk': + vercode, version = get_metadata_from_apk(app, build, src) + if version != build.versionName or vercode != build.versionCode: + raise fdroidserver.exception.BuildException( + ( + "Unexpected version/version code in output;" + " APK: '%s' / '%d', " + " Expected: '%s' / '%d'" + ) + % (version, vercode, build.versionName, build.versionCode) + ) + + +def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball=False): + tarname = fdroidserver.common.getsrcname(app, build) + + # Copy the unsigned APK to our destination directory for further + # processing (by publish.py)... + dest = os.path.join( + output_dir, + fdroidserver.common.get_release_filename( + app, build, fdroidserver.common.get_file_extension(src) + ), + ) + shutil.copyfile(src, dest) + + # Move the source tarball into the output directory... + if output_dir != tmp_dir and not notarball: + shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname)) + + +def run_this_build(config, options, package_name, version_code): + """run build for one specific version of an app localy + + :raises: various exceptions in case and of the pre-required conditions for the requested build are not met + """ + + app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) + + # not sure if this makes any sense to change open file limits since we know + # that this script will only ever build one app + rlimit_check(1) + + logging.info( + "Building version %s (%s) of %s" + % (build.versionName, build.versionCode, app.id) + ) + + # init fdroid Popen wrapper + fdroidserver.common.set_FDroidPopen_env(build) + p, gradletasks, flavours_cmd = init_build_subprocess(app, build, config) + + sanitize_build_dir(app) + + # this is where we'd call scanner.scan_source() in old build.py + + # create tarball before building + # consider this optional? + tmp_dir = pathlib.Path("./tmp") + tmp_dir.mkdir(exist_ok=True) + make_tarball(app, build, tmp_dir) + + # Run a build command if one is required... + execute_build_commands(app, build) + + # Build native stuff if required... + execute_buildjni_commands(app, build) + + # Build the release... + p = execute_build___(app, build, config, gradletasks) + collect_build_output(app, build, p, flavours_cmd) + buildbuildbuild___(app, build, config, gradletasks) + check_build_success(app, build, p) + src = collect_output_again(app, build, p, flavours_cmd) + + # Run a postbuild command if one is required... + execute_postbuild(app, build, src) + + validate_build_artifacts(app, build, src) + + # this is where we'd call scanner.scan_binary() in old build.py + + move_build_output(app, build, src, tmp_dir) + +def main(): + parser = argparse.ArgumentParser( + description=_( + "Build one specific in app. This command " + "assumes all required build tools are installed and " + "configured." + ) + ) + parser.add_argument( + "APP_VERSION", help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)") + ) + + # fdroid args/opts boilerplate + fdroidserver.common.setup_global_opts(parser) + options = fdroidserver.common.parse_args(parser) + config = fdroidserver.common.get_config() + + try: + # read build target package name and version code from CLI arguments + package_name, version_code = fdroidserver.common.split_pkg_arg( + options.APP_VERSION + ) + # trigger the build + run_this_build(config, options, package_name, version_code) + except Exception as e: + if options.verbose: + traceback.print_exc() + else: + print(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5a546386..52e65aa4 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -870,6 +870,27 @@ def get_local_metadata_files(): return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') +def split_pkg_arg(appid_versionCode_pair): + """Split 'appid:versionCode' pair into 2 separate values safely. + + :raises ValueError: if argument is not parseable + :return: (appid, versionCode) tuple with the 2 parsed values + """ + tokens = appid_versionCode_pair.split(":") + if len(tokens) != 2: + raise ValueError( + _("'{}' is not a valid pair of the form appId:versionCode pair").format( + appid_versionCode_pair + ) + ) + if not is_valid_package_name(tokens[0]): + raise ValueError( + _("'{}' does not start with a valid appId").format(appid_versionCode_pair) + ) + versionCode = version_code_string_to_int(tokens[1]) + return tokens[0], versionCode + + def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False): """No summary. diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ce7e5ee8..74a663c1 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -607,6 +607,45 @@ def read_metadata(appids={}, sort_by_time=False): return apps +def read_build_metadata(package_name, version_code, check_disabled=True): + """read 1 single metadata file from the file system + 1 singled out build + + Parameters + ---------- + package_name + appid of the metadata supposed to be loaded (e.g. 'org.fdroid.fdroid') + version_code + android integer version identifier of a build (e.g. 1234) + check_disabled + If True this function will raise an exception in case the disabled flag + on the build or the metadata file is set. + + Returns + ------- + Tuple + A tuple of (metadata, build) dictionsaries, containing the pared data from + the metadata file. + + Raises + ------ + MetaDataException + If parsing or selecting the requested version fails. + """ + m = read_metadata({package_name: version_code}) + if len(m) != 1 or package_name not in m: + raise MetaDataException(f"Could not read metadata for '{package_name}:{version_code}' (metadata file might not be present or correct)") + if check_disabled and m[package_name].get("Disabled"): + raise MetaDataException(f"'{package_name}' disbaled in metadata file.") + if "Builds" not in m[package_name]: + raise MetaDataException(f"Cound not find 'Builds' section in '{package_name}' metadata file") + for b in m[package_name]["Builds"]: + if b['versionCode'] == version_code: + if check_disabled and b.get('disable'): + raise MetaDataException(f"'{package_name}:{version_code}' disabled in metadata file") + return m[package_name], b + raise MetaDataException(f"Could not find '{version_code}' in 'Builds' section of 'package_name' metadata file") + + def parse_metadata(metadatapath): """Parse metadata file, also checking the source repo for .fdroid.yml.