From ffa9e74d3fff36226e640f5e490bf5ed8453373f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 16 May 2024 15:13:27 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=A7=AD=20stated=20splitting=20out=20b?= =?UTF-8?q?uild=5Flocal=20from=20build.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This copies build_local() from build.py into newly created (hidden) subcommands: `build_local_prepare` and `build_local_run`. It also breaks up the long body of the build_local function into smaller functions. --- fdroidserver/__main__.py | 17 +- fdroidserver/build_local_prepare.py | 154 +++++ fdroidserver/build_local_run.py | 871 ++++++++++++++++++++++++++++ fdroidserver/common.py | 21 + fdroidserver/metadata.py | 39 ++ 5 files changed, 1100 insertions(+), 2 deletions(-) create mode 100644 fdroidserver/build_local_prepare.py create mode 100644 fdroidserver/build_local_run.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 14813fa1..073437f8 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -56,6 +56,14 @@ COMMANDS = OrderedDict([ ]) +# commands that are not advertised as public api, intended for the use-case +# of breaking down builds in buildbot into smaller steps +COMMANDS_INTERNAL = [ + "build_local_prepare", + "build_local_run", +] + + def print_help(available_plugins=None): print(_("usage: ") + _("fdroid [] [-h|--help|--version|]")) print("") @@ -136,7 +144,12 @@ def main(): sys.exit(0) command = sys.argv[1] - if command not in COMMANDS and command not in available_plugins: + command_not_found = ( + command not in COMMANDS + and command not in COMMANDS_INTERNAL + and command not in available_plugins + ) + if command_not_found: if command in ('-h', '--help'): print_help(available_plugins=available_plugins) sys.exit(0) @@ -186,7 +199,7 @@ def main(): sys.argv[0] += ' ' + command del sys.argv[1] - if command in COMMANDS.keys(): + if command in COMMANDS.keys() or command in COMMANDS_INTERNAL: # import is named import_subcommand internally b/c import is reserved by Python command = 'import_subcommand' if command == 'import' else command mod = __import__('fdroidserver.' + command, None, None, [command]) diff --git a/fdroidserver/build_local_prepare.py b/fdroidserver/build_local_prepare.py new file mode 100644 index 00000000..a495d0a8 --- /dev/null +++ b/fdroidserver/build_local_prepare.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +# build.py - part of the FDroid server tools +# Copyright (C) 2024, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import os +import pathlib +import logging +import argparse + +from fdroidserver import _ +import fdroidserver.common + + +def sudo_run(app, build): + # before doing anything, run the sudo commands to setup the VM + if build.sudo: + logging.info("Running 'sudo' commands in %s" % os.getcwd()) + + p = fdroidserver.common.FDroidPopen( + [ + 'sudo', + 'DEBIAN_FRONTEND=noninteractive', + 'bash', + '-e', + '-u', + '-o', + 'pipefail', + '-x', + '-c', + '; '.join(build.sudo), + ] + ) + if p.returncode != 0: + raise BuildException( + "Error running sudo command for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def sudo_lock_root(app, build): + p = fdroidserver.common.FDroidPopen(['sudo', 'passwd', '--lock', 'root']) + if p.returncode != 0: + raise BuildException( + "Error locking root account for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def sudo_uninstall(app, build): + p = fdroidserver.common.FDroidPopen( + ['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo'] + ) + if p.returncode != 0: + raise BuildException( + "Error removing sudo for %s:%s" % (app.id, build.versionName), p.output + ) + + +def log_tools_version(app, build, log_dir): + log_path = os.path.join( + log_dir, fdroidserver.common.get_toolsversion_logname(app, build) + ) + with open(log_path, 'w') as f: + f.write(fdroidserver.common.get_android_tools_version_log()) + + +def main(): + parser = argparse.ArgumentParser( + description=_( + "Download source code and initialize build environment " + "for one specific build" + ), + ) + parser.add_argument( + "--sudo-run", + action="store_true", + default=False, + help=_("run commands listed in sudo-metadata"), + ) + parser.add_argument( + "--sudo-uninstall", + action="store_true", + default=False, + help=_("uninstall sudo executing sudo-metadata"), + ) + parser.add_argument( + "--sudo-lock-root", + action="store_true", + default=False, + help=_("lock root user account"), + ) + parser.add_argument( + "APP_VERSION", + help=_("app id and version code tuple 'APPID:VERCODE'"), + ) + + # fdroid args/opts boilerplate + fdroidserver.common.setup_global_opts(parser) + options = fdroidserver.common.parse_args(parser) + config = fdroidserver.common.get_config() + + package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION) + app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) + + # prepare folders for git/vcs checkout + vcs, build_dir = fdroidserver.common.setup_vcs(app) + srclib_dir = pathlib.Path('./build/srclib') + extlib_dir = pathlib.Path('./build/extlib') + log_dir = pathlib.Path('./logs') + output_dir = pathlib.Path('./unsigned') + for d in (srclib_dir, extlib_dir, log_dir, output_dir): + d.mkdir(exist_ok=True, parents=True) + + # run sudo stuff + fdroidserver.common.set_FDroidPopen_env(build) + if options.sudo_run: + sudo_run(app, build) + if options.sudo_lock_root: + sudo_lock_root(app, build) + if options.sudo_uninstall: + sudo_uninstall(app, build) + + # TODO: in the past this was only logged when running as 'fdroid build + # --onserver' is this this output still valuable or can we remove it? + log_tools_version(app, build, log_dir) + + # do git/vcs checkout + fdroidserver.common.prepare_source( + vcs, app, build, build_dir, str(srclib_dir), str(extlib_dir) + ) + + # prepare for running cli commands + # NOTE: unclear if this is required here + # ndk_path = get_ndk_path(build) + # fdroidserver.common.set_FDroidPopen_env(build) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/build_local_run.py b/fdroidserver/build_local_run.py new file mode 100644 index 00000000..10f7caf5 --- /dev/null +++ b/fdroidserver/build_local_run.py @@ -0,0 +1,871 @@ +import os +import sys +import glob +import shutil +import gettext +import logging +import pathlib +import tarfile +import argparse +import traceback + +import fdroidserver.common +import fdroidserver.metadata +import fdroidserver.exception + +from fdroidserver import _ + + +def rlimit_check(apps_count): + """make sure enough open files are allowed to process everything""" + try: + import resource # not available on Windows + + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if apps_count > soft: + try: + soft = apps_count * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug( + _('Set open file limit to {integer}').format(integer=soft) + ) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) + except ImportError: + pass + + +def get_ndk_path(build): + ndk_path = build.ndk_path() + if build.ndk or (build.buildjni and build.buildjni != ['no']): + if not ndk_path: + logging.warning("Android NDK version '%s' could not be found!" % build.ndk) + logging.warning("Configured versions:") + for k, v in config['ndk_paths'].items(): + if k.endswith("_orig"): + continue + logging.warning(" %s: %s" % (k, v)) + if onserver: + fdroidserver.common.auto_install_ndk(build) + else: + raise fdroidserver.exception.FDroidException() + elif not os.path.isdir(ndk_path): + logging.critical("Android NDK '%s' is not a directory!" % ndk_path) + raise fdroidserver.exception.FDroidException() + + +def get_build_root_dir(app, build): + if build.subdir: + return os.path.join(fdroidserver.common.get_build_dir(app), build.subdir) + return fdroidserver.common.get_build_dir(app) + + +def transform_first_char(string, method): + """Use method() on the first character of string.""" + if len(string) == 0: + return string + if len(string) == 1: + return method(string) + return method(string[0]) + string[1:] + + +def init_build_subprocess(app, build, config): + # We need to clean via the build tool in case the binary dirs are + # different from the default ones + root_dir = get_build_root_dir(app, build) + + p = None + gradletasks = [] + flavours_cmd = None + + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Cleaning Maven project...") + cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']] + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + maven_dir = os.path.normpath(maven_dir) + else: + maven_dir = root_dir + + p = fdroidserver.common.FDroidPopen(cmd, cwd=maven_dir) + + elif bmethod == 'gradle': + + logging.info("Cleaning Gradle project...") + + if build.preassemble: + gradletasks += build.preassemble + + flavours = build.gradle + if flavours == ['yes']: + flavours = [] + + flavours_cmd = ''.join( + [transform_first_char(flav, str.upper) for flav in flavours] + ) + + gradletasks += ['assemble' + flavours_cmd + 'Release'] + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += ['clean'] + p = fdroidserver.common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Cleaning Ant project...") + p = fdroidserver.common.FDroidPopen(['ant', 'clean'], cwd=root_dir) + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error cleaning %s:%s" % (app.id, build.versionName), p.output + ) + + return p, gradletasks, flavours_cmd + + +def sanitize_build_dir(app): + build_dir = fdroidserver.common.get_build_dir(app) + for root, dirs, files in os.walk(build_dir): + + def del_dirs(dl): + for d in dl: + shutil.rmtree(os.path.join(root, d), ignore_errors=True) + + def del_files(fl): + for f in fl: + if f in files: + os.remove(os.path.join(root, f)) + + if any( + f in files + for f in [ + 'build.gradle', + 'build.gradle.kts', + 'settings.gradle', + 'settings.gradle.kts', + ] + ): + # Even when running clean, gradle stores task/artifact caches in + # .gradle/ as binary files. To avoid overcomplicating the scanner, + # manually delete them, just like `gradle clean` should have removed + # the build/* dirs. + del_dirs( + [ + os.path.join('build', 'android-profile'), + os.path.join('build', 'generated'), + os.path.join('build', 'intermediates'), + os.path.join('build', 'outputs'), + os.path.join('build', 'reports'), + os.path.join('build', 'tmp'), + os.path.join('buildSrc', 'build'), + '.gradle', + ] + ) + del_files(['gradlew', 'gradlew.bat']) + + if 'pom.xml' in files: + del_dirs(['target']) + + if any( + f in files for f in ['ant.properties', 'project.properties', 'build.xml'] + ): + del_dirs(['bin', 'gen']) + + if 'jni' in dirs: + del_dirs(['obj']) + + +def make_tarball(app, build, tmp_dir): + build_dir = fdroidserver.common.get_build_dir(app) + # Build the source tarball right before we build the release... + logging.info("Creating source tarball...") + tarname = fdroidserver.common.getsrcname(app, build) + tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") + + def tarexc(t): + return ( + None + if any(t.name.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) + else t + ) + + tarball.add(build_dir, tarname, filter=tarexc) + tarball.close() + + +def execute_build_commands(app, build): + build_dir = fdroidserver.common.get_build_dir(app) + srclibpaths = get_srclibpaths(build, build_dir) + if build.build: + logging.info("Running 'build' commands in %s" % root_dir) + cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build) + + # Substitute source library paths into commands... + for name, number, libpath in srclibpaths: + cmd = cmd.replace('$$' + name + '$$', os.path.join(os.getcwd(), libpath)) + + p = fdroidserver.common.FDroidPopen( + ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir + ) + + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error running build command for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def get_srclibpaths(build, srclib_dir): + srclibpaths = [] + if build.srclibs: + logging.info("Collecting source libraries") + for lib in build.srclibs: + srclibpaths.append( + fdroidserver.common.getsrclib( + lib, srclib_dir, preponly=onserver, refresh=refresh, build=build + ) + ) + return srclibpaths + + +def execute_buildjni_commands(app, build): + root_dir = get_build_root_dir(app, build) + ndk_path = get_ndk_path(build) + if build.buildjni and build.buildjni != ['no']: + logging.info("Building the native code") + jni_components = build.buildjni + + if jni_components == ['yes']: + jni_components = [''] + cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"] + for d in jni_components: + if d: + logging.info("Building native code in '%s'" % d) + else: + logging.info("Building native code in the main project") + manifest = os.path.join(root_dir, d, 'AndroidManifest.xml') + if os.path.exists(manifest): + # Read and write the whole AM.xml to fix newlines and avoid + # the ndk r8c or later 'wordlist' errors. The outcome of this + # under gnu/linux is the same as when using tools like + # dos2unix, but the native python way is faster and will + # work in non-unix systems. + manifest_text = open(manifest, 'U').read() + open(manifest, 'w').write(manifest_text) + # In case the AM.xml read was big, free the memory + del manifest_text + p = fdroidserver.common.FDroidPopen(cmd, cwd=os.path.join(root_dir, d)) + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "NDK build failed for %s:%s" % (app.id, build.versionName), p.output + ) + + +def execute_build___(app, build, config, gradletasks): + build_dir = fdroidserver.common.get_build_dir(app) + root_dir = get_build_root_dir(app, build) + + p = None + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Building Maven project...") + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + else: + maven_dir = root_dir + + mvncmd = [ + config['mvn3'], + '-Dandroid.sdk.path=' + config['sdk_path'], + '-Dmaven.jar.sign.skip=true', + '-Dmaven.test.skip=true', + '-Dandroid.sign.debug=false', + '-Dandroid.release=true', + 'package', + ] + if build.target: + target = build.target.split('-')[1] + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(root_dir, 'pom.xml'), + ) + if '@' in build.maven: + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(maven_dir, 'pom.xml'), + ) + + p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir) + + bindir = os.path.join(root_dir, 'target') + + elif bmethod == 'gradle': + logging.info("Building Gradle project...") + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += gradletasks + + p = fdroidserver.common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Building Ant project...") + cmd = ['ant'] + if build.antcommands: + cmd += build.antcommands + else: + cmd += ['release'] + p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir) + + bindir = os.path.join(root_dir, 'bin') + + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + + commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output, + ) + logging.info( + "Successfully built version {versionName} of {appid} from {commit_id}".format( + versionName=build.versionName, appid=app.id, commit_id=commit_id + ) + ) + + return p + + +def collect_build_output(app, build, p, flavours_cmd): + root_dir = get_build_root_dir(app, build) + + omethod = build.output_method() + if omethod == 'maven': + stdout_apk = '\n'.join( + [ + line + for line in p.output.splitlines() + if any(a in line for a in ('.apk', '.ap_', '.jar')) + ] + ) + m = re.match( + r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M + ) + if not m: + m = re.match( + r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", + stdout_apk, + re.S | re.M, + ) + if not m: + m = re.match( + r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + + bindir + + r'/([^/]+)\.ap[_k][,\]]', + stdout_apk, + re.S | re.M, + ) + + if not m: + m = re.match( + r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", + stdout_apk, + re.S | re.M, + ) + if not m: + raise fdroidserver.exception.BuildException('Failed to find output') + src = m.group(1) + src = os.path.join(bindir, src) + '.apk' + + elif omethod == 'gradle': + src = None + apk_dirs = [ + # gradle plugin >= 3.0 + os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'), + # gradle plugin < 3.0 and >= 0.11 + os.path.join(root_dir, 'build', 'outputs', 'apk'), + # really old path + os.path.join(root_dir, 'build', 'apk'), + ] + # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in + # a subdirectory corresponding to the flavour command used, but with different + # capitalization. + if flavours_cmd: + apk_dirs.append( + os.path.join( + root_dir, + 'build', + 'outputs', + 'apk', + transform_first_char(flavours_cmd, str.lower), + 'release', + ) + ) + for apks_dir in apk_dirs: + for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: + apks = glob.glob(os.path.join(apks_dir, apkglob)) + + if len(apks) > 1: + raise fdroidserver.exception.BuildException( + 'More than one resulting apks found in %s' % apks_dir, + '\n'.join(apks), + ) + if len(apks) == 1: + src = apks[0] + break + if src is not None: + break + + if src is None: + raise fdroidserver.exception.BuildException('Failed to find any output apks') + + elif omethod == 'ant': + stdout_apk = '\n'.join( + [line for line in p.output.splitlines() if '.apk' in line] + ) + src = re.match( + r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M + ).group(1) + src = os.path.join(bindir, src) + elif omethod == 'raw': + output_path = fdroidserver.common.replace_build_vars(build.output, build) + globpath = os.path.join(root_dir, output_path) + apks = glob.glob(globpath) + if len(apks) > 1: + raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) + if len(apks) < 1: + raise fdroidserver.exception.BuildException('No apks match %s' % globpath) + src = os.path.normpath(apks[0]) + + +def buildbuildbuild___(app, build, config, gradletasks): + root_dir = get_build_root_dir(app, build) + + bmethod = build.build_method() + if bmethod == 'maven': + logging.info("Building Maven project...") + + if '@' in build.maven: + maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) + else: + maven_dir = root_dir + + mvncmd = [ + config['mvn3'], + '-Dandroid.sdk.path=' + config['sdk_path'], + '-Dmaven.jar.sign.skip=true', + '-Dmaven.test.skip=true', + '-Dandroid.sign.debug=false', + '-Dandroid.release=true', + 'package', + ] + if build.target: + target = build.target.split('-')[1] + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(root_dir, 'pom.xml'), + ) + if '@' in build.maven: + fdroidserver.common.regsub_file( + r'[0-9]*', + r'%s' % target, + os.path.join(maven_dir, 'pom.xml'), + ) + + p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir) + + bindir = os.path.join(root_dir, 'target') + + elif bmethod == 'gradle': + logging.info("Building Gradle project...") + + cmd = [config['gradle']] + if build.gradleprops: + cmd += ['-P' + kv for kv in build.gradleprops] + + cmd += gradletasks + + p = fdroidserver.common.FDroidPopen( + cmd, + cwd=root_dir, + envs={ + "GRADLE_VERSION_DIR": config['gradle_version_dir'], + "CACHEDIR": config['cachedir'], + }, + ) + + elif bmethod == 'ant': + logging.info("Building Ant project...") + cmd = ['ant'] + if build.antcommands: + cmd += build.antcommands + else: + cmd += ['release'] + p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir) + + bindir = os.path.join(root_dir, 'bin') + + +def check_build_success(app, build, p): + build_dir = fdroidserver.common.get_build_dir(app) + + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + + commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output, + ) + logging.info( + "Successfully built version {versionName} of {appid} from {commit_id}".format( + versionName=build.versionName, appid=app.id, commit_id=commit_id + ) + ) + + +def collect_output_again(app, build, p, flavours_cmd): + root_dir = get_build_root_dir(app, build) + + omethod = build.output_method() + src = None + if omethod == 'maven': + stdout_apk = '\n'.join( + [ + line + for line in p.output.splitlines() + if any(a in line for a in ('.apk', '.ap_', '.jar')) + ] + ) + m = re.match( + r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M + ) + if not m: + m = re.match( + r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", + stdout_apk, + re.S | re.M, + ) + if not m: + m = re.match( + r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + + bindir + + r'/([^/]+)\.ap[_k][,\]]', + stdout_apk, + re.S | re.M, + ) + + if not m: + m = re.match( + r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", + stdout_apk, + re.S | re.M, + ) + if not m: + raise fdroidserver.exception.BuildException('Failed to find output') + src = m.group(1) + src = os.path.join(bindir, src) + '.apk' + + elif omethod == 'gradle': + src = None + apk_dirs = [ + # gradle plugin >= 3.0 + os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'), + # gradle plugin < 3.0 and >= 0.11 + os.path.join(root_dir, 'build', 'outputs', 'apk'), + # really old path + os.path.join(root_dir, 'build', 'apk'), + ] + # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in + # a subdirectory corresponding to the flavour command used, but with different + # capitalization. + if flavours_cmd: + apk_dirs.append( + os.path.join( + root_dir, + 'build', + 'outputs', + 'apk', + transform_first_char(flavours_cmd, str.lower), + 'release', + ) + ) + for apks_dir in apk_dirs: + for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: + apks = glob.glob(os.path.join(apks_dir, apkglob)) + + if len(apks) > 1: + raise fdroidserver.exception.BuildException( + 'More than one resulting apks found in %s' % apks_dir, + '\n'.join(apks), + ) + if len(apks) == 1: + src = apks[0] + break + if src is not None: + break + + if src is None: + raise fdroidserver.exception.BuildException('Failed to find any output apks') + + elif omethod == 'ant': + stdout_apk = '\n'.join( + [line for line in p.output.splitlines() if '.apk' in line] + ) + src = re.match( + r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M + ).group(1) + src = os.path.join(bindir, src) + elif omethod == 'raw': + output_path = fdroidserver.common.replace_build_vars(build.output, build) + globpath = os.path.join(root_dir, output_path) + apks = glob.glob(globpath) + if len(apks) > 1: + raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) + if len(apks) < 1: + raise fdroidserver.exception.BuildException('No apks match %s' % globpath) + src = os.path.normpath(apks[0]) + return src + + +def execute_postbuild(app, build, src): + build_dir = fdroidserver.common.get_build_dir(app) + root_dir = get_build_root_dir(app, build) + srclibpaths = get_srclibpaths(build, build_dir) + + if build.postbuild: + logging.info(f"Running 'postbuild' commands in {root_dir}") + cmd = fdroidserver.common.replace_config_vars("; ".join(build.postbuild), build) + + # Substitute source library paths into commands... + for name, number, libpath in srclibpaths: + cmd = cmd.replace(f"$${name}$$", str(Path.cwd() / libpath)) + + cmd = cmd.replace('$$OUT$$', str(Path(src).resolve())) + + p = fdroidserver.common.FDroidPopen( + ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir + ) + + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error running postbuild command for " f"{app.id}:{build.versionName}", + p.output, + ) + + +def get_metadata_from_apk(app, build, apkfile): + """Get the required metadata from the built APK. + + VersionName is allowed to be a blank string, i.e. '' + + Parameters + ---------- + app + The app metadata used to build the APK. + build + The build that resulted in the APK. + apkfile + The path of the APK file. + + Returns + ------- + versionCode + The versionCode from the APK or from the metadata is build.novcheck is + set. + versionName + The versionName from the APK or from the metadata is build.novcheck is + set. + + Raises + ------ + :exc:`~fdroidserver.exception.BuildException` + If native code should have been built but was not packaged, no version + information or no package ID could be found or there is a mismatch + between the package ID in the metadata and the one found in the APK. + """ + appid, versionCode, versionName = fdroidserver.common.get_apk_id(apkfile) + native_code = fdroidserver.common.get_native_code(apkfile) + + if build.buildjni and build.buildjni != ['no'] and not native_code: + raise fdroidserver.exception.BuildException("Native code should have been built but none was packaged") + if build.novcheck: + versionCode = build.versionCode + versionName = build.versionName + if not versionCode or versionName is None: + raise fdroidserver.exception.BuildException("Could not find version information in build in output") + if not appid: + raise fdroidserver.exception.BuildException("Could not find package ID in output") + if appid != app.id: + raise fdroidserver.exception.BuildException( + "Wrong package ID - build " + appid + " but expected " + app.id + ) + + return versionCode, versionName + + +def validate_build_artifacts(app, build, src): + # Make sure it's not debuggable... + if fdroidserver.common.is_debuggable_or_testOnly(src): + raise fdroidserver.exception.BuildException( + "%s: debuggable or testOnly set in AndroidManifest.xml" % src + ) + + # By way of a sanity check, make sure the version and version + # code in our new APK match what we expect... + logging.debug("Checking " + src) + if not os.path.exists(src): + raise fdroidserver.exception.BuildException("Unsigned APK is not at expected location of " + src) + + if fdroidserver.common.get_file_extension(src) == 'apk': + vercode, version = get_metadata_from_apk(app, build, src) + if version != build.versionName or vercode != build.versionCode: + raise fdroidserver.exception.BuildException( + ( + "Unexpected version/version code in output;" + " APK: '%s' / '%d', " + " Expected: '%s' / '%d'" + ) + % (version, vercode, build.versionName, build.versionCode) + ) + + +def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball=False): + tarname = fdroidserver.common.getsrcname(app, build) + + # Copy the unsigned APK to our destination directory for further + # processing (by publish.py)... + dest = os.path.join( + output_dir, + fdroidserver.common.get_release_filename( + app, build, fdroidserver.common.get_file_extension(src) + ), + ) + shutil.copyfile(src, dest) + + # Move the source tarball into the output directory... + if output_dir != tmp_dir and not notarball: + shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname)) + + +def run_this_build(config, options, package_name, version_code): + """run build for one specific version of an app localy + + :raises: various exceptions in case and of the pre-required conditions for the requested build are not met + """ + + app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) + + # not sure if this makes any sense to change open file limits since we know + # that this script will only ever build one app + rlimit_check(1) + + logging.info( + "Building version %s (%s) of %s" + % (build.versionName, build.versionCode, app.id) + ) + + # init fdroid Popen wrapper + fdroidserver.common.set_FDroidPopen_env(build) + p, gradletasks, flavours_cmd = init_build_subprocess(app, build, config) + + sanitize_build_dir(app) + + # this is where we'd call scanner.scan_source() in old build.py + + # create tarball before building + # consider this optional? + tmp_dir = pathlib.Path("./tmp") + tmp_dir.mkdir(exist_ok=True) + make_tarball(app, build, tmp_dir) + + # Run a build command if one is required... + execute_build_commands(app, build) + + # Build native stuff if required... + execute_buildjni_commands(app, build) + + # Build the release... + p = execute_build___(app, build, config, gradletasks) + collect_build_output(app, build, p, flavours_cmd) + buildbuildbuild___(app, build, config, gradletasks) + check_build_success(app, build, p) + src = collect_output_again(app, build, p, flavours_cmd) + + # Run a postbuild command if one is required... + execute_postbuild(app, build, src) + + validate_build_artifacts(app, build, src) + + # this is where we'd call scanner.scan_binary() in old build.py + + move_build_output(app, build, src, tmp_dir) + +def main(): + parser = argparse.ArgumentParser( + description=_( + "Build one specific in app. This command " + "assumes all required build tools are installed and " + "configured." + ) + ) + parser.add_argument( + "APP_VERSION", help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)") + ) + + # fdroid args/opts boilerplate + fdroidserver.common.setup_global_opts(parser) + options = fdroidserver.common.parse_args(parser) + config = fdroidserver.common.get_config() + + try: + # read build target package name and version code from CLI arguments + package_name, version_code = fdroidserver.common.split_pkg_arg( + options.APP_VERSION + ) + # trigger the build + run_this_build(config, options, package_name, version_code) + except Exception as e: + if options.verbose: + traceback.print_exc() + else: + print(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5a546386..52e65aa4 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -870,6 +870,27 @@ def get_local_metadata_files(): return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') +def split_pkg_arg(appid_versionCode_pair): + """Split 'appid:versionCode' pair into 2 separate values safely. + + :raises ValueError: if argument is not parseable + :return: (appid, versionCode) tuple with the 2 parsed values + """ + tokens = appid_versionCode_pair.split(":") + if len(tokens) != 2: + raise ValueError( + _("'{}' is not a valid pair of the form appId:versionCode pair").format( + appid_versionCode_pair + ) + ) + if not is_valid_package_name(tokens[0]): + raise ValueError( + _("'{}' does not start with a valid appId").format(appid_versionCode_pair) + ) + versionCode = version_code_string_to_int(tokens[1]) + return tokens[0], versionCode + + def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False): """No summary. diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ce7e5ee8..74a663c1 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -607,6 +607,45 @@ def read_metadata(appids={}, sort_by_time=False): return apps +def read_build_metadata(package_name, version_code, check_disabled=True): + """read 1 single metadata file from the file system + 1 singled out build + + Parameters + ---------- + package_name + appid of the metadata supposed to be loaded (e.g. 'org.fdroid.fdroid') + version_code + android integer version identifier of a build (e.g. 1234) + check_disabled + If True this function will raise an exception in case the disabled flag + on the build or the metadata file is set. + + Returns + ------- + Tuple + A tuple of (metadata, build) dictionsaries, containing the pared data from + the metadata file. + + Raises + ------ + MetaDataException + If parsing or selecting the requested version fails. + """ + m = read_metadata({package_name: version_code}) + if len(m) != 1 or package_name not in m: + raise MetaDataException(f"Could not read metadata for '{package_name}:{version_code}' (metadata file might not be present or correct)") + if check_disabled and m[package_name].get("Disabled"): + raise MetaDataException(f"'{package_name}' disbaled in metadata file.") + if "Builds" not in m[package_name]: + raise MetaDataException(f"Cound not find 'Builds' section in '{package_name}' metadata file") + for b in m[package_name]["Builds"]: + if b['versionCode'] == version_code: + if check_disabled and b.get('disable'): + raise MetaDataException(f"'{package_name}:{version_code}' disabled in metadata file") + return m[package_name], b + raise MetaDataException(f"Could not find '{version_code}' in 'Builds' section of 'package_name' metadata file") + + def parse_metadata(metadatapath): """Parse metadata file, also checking the source repo for .fdroid.yml. From 33687cf41bf613a73ddb490a0a1696b992f80336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 5 Jun 2024 17:18:20 +0200 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=A6=B9=20add=20build=5Flocal=5Fsudo?= =?UTF-8?q?=20subcommand?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This subcommand takes following actions: * executes sudo commands form metadata * removes sudo from your system * locks root account Only run in a vm/container!!! --- fdroidserver/__main__.py | 1 + fdroidserver/build_local_sudo.py | 104 +++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 fdroidserver/build_local_sudo.py diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 073437f8..b45d2938 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -61,6 +61,7 @@ COMMANDS = OrderedDict([ COMMANDS_INTERNAL = [ "build_local_prepare", "build_local_run", + "build_local_sudo", ] diff --git a/fdroidserver/build_local_sudo.py b/fdroidserver/build_local_sudo.py new file mode 100644 index 00000000..e878edae --- /dev/null +++ b/fdroidserver/build_local_sudo.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# +# build.py - part of the FDroid server tools +# Copyright (C) 2024, Michael Pöhn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import os +import logging +import argparse + +from fdroidserver import _ +import fdroidserver.common + + +def sudo_run(app, build): + # before doing anything, run the sudo commands to setup the VM + if build.sudo: + logging.info("Running 'sudo' commands in %s" % os.getcwd()) + + p = fdroidserver.common.FDroidPopen( + [ + 'sudo', + 'DEBIAN_FRONTEND=noninteractive', + 'bash', + '-e', + '-u', + '-o', + 'pipefail', + '-x', + '-c', + '; '.join(build.sudo), + ] + ) + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error running sudo command for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def sudo_lock_root(app, build): + p = fdroidserver.common.FDroidPopen(['sudo', 'passwd', '--lock', 'root']) + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error locking root account for %s:%s" % (app.id, build.versionName), + p.output, + ) + + +def sudo_uninstall(app, build): + p = fdroidserver.common.FDroidPopen( + ['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo'] + ) + if p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Error removing sudo for %s:%s" % (app.id, build.versionName), p.output + ) + + +def main(): + parser = argparse.ArgumentParser( + description=_( + """Run sudo commands """ + ), + ) + parser.add_argument( + "APP_VERSION", + help=_("app id and version code tuple 'APPID:VERCODE'"), + ) + + # fdroid args/opts boilerplate + fdroidserver.common.setup_global_opts(parser) + options = fdroidserver.common.parse_args(parser) + config = fdroidserver.common.get_config() + config # silcense pyflakes + + package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION) + app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) + + + # intialize FDroidPopen + # TODO: remove once FDroidPopen is replaced with vm/container exec + fdroidserver.common.set_FDroidPopen_env(build) + + # run sudo stuff + sudo_run(app, build) + sudo_lock_root(app, build) + sudo_uninstall(app, build) + +if __name__ == "__main__": + main() From 1bde8d27b02f314b1d4828e8c96c4ffc265d2a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 6 Jun 2024 11:50:16 +0200 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=A7=B9=20remove=20sudo=20code=20from?= =?UTF-8?q?=20build=5Flocal=5Fprepare?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/build_local_prepare.py | 78 ++--------------------------- 1 file changed, 4 insertions(+), 74 deletions(-) diff --git a/fdroidserver/build_local_prepare.py b/fdroidserver/build_local_prepare.py index a495d0a8..7ac01631 100644 --- a/fdroidserver/build_local_prepare.py +++ b/fdroidserver/build_local_prepare.py @@ -19,58 +19,13 @@ import os import pathlib -import logging +# import logging import argparse from fdroidserver import _ import fdroidserver.common -def sudo_run(app, build): - # before doing anything, run the sudo commands to setup the VM - if build.sudo: - logging.info("Running 'sudo' commands in %s" % os.getcwd()) - - p = fdroidserver.common.FDroidPopen( - [ - 'sudo', - 'DEBIAN_FRONTEND=noninteractive', - 'bash', - '-e', - '-u', - '-o', - 'pipefail', - '-x', - '-c', - '; '.join(build.sudo), - ] - ) - if p.returncode != 0: - raise BuildException( - "Error running sudo command for %s:%s" % (app.id, build.versionName), - p.output, - ) - - -def sudo_lock_root(app, build): - p = fdroidserver.common.FDroidPopen(['sudo', 'passwd', '--lock', 'root']) - if p.returncode != 0: - raise BuildException( - "Error locking root account for %s:%s" % (app.id, build.versionName), - p.output, - ) - - -def sudo_uninstall(app, build): - p = fdroidserver.common.FDroidPopen( - ['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo'] - ) - if p.returncode != 0: - raise BuildException( - "Error removing sudo for %s:%s" % (app.id, build.versionName), p.output - ) - - def log_tools_version(app, build, log_dir): log_path = os.path.join( log_dir, fdroidserver.common.get_toolsversion_logname(app, build) @@ -86,24 +41,6 @@ def main(): "for one specific build" ), ) - parser.add_argument( - "--sudo-run", - action="store_true", - default=False, - help=_("run commands listed in sudo-metadata"), - ) - parser.add_argument( - "--sudo-uninstall", - action="store_true", - default=False, - help=_("uninstall sudo executing sudo-metadata"), - ) - parser.add_argument( - "--sudo-lock-root", - action="store_true", - default=False, - help=_("lock root user account"), - ) parser.add_argument( "APP_VERSION", help=_("app id and version code tuple 'APPID:VERCODE'"), @@ -114,6 +51,8 @@ def main(): options = fdroidserver.common.parse_args(parser) config = fdroidserver.common.get_config() + config # silence pyflakes + package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION) app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) @@ -126,17 +65,8 @@ def main(): for d in (srclib_dir, extlib_dir, log_dir, output_dir): d.mkdir(exist_ok=True, parents=True) - # run sudo stuff - fdroidserver.common.set_FDroidPopen_env(build) - if options.sudo_run: - sudo_run(app, build) - if options.sudo_lock_root: - sudo_lock_root(app, build) - if options.sudo_uninstall: - sudo_uninstall(app, build) - # TODO: in the past this was only logged when running as 'fdroid build - # --onserver' is this this output still valuable or can we remove it? + # --onserver', is this output still valuable or can we remove it? log_tools_version(app, build, log_dir) # do git/vcs checkout From 629fd1a204ec6efb2b4e9ff2f9859f68c4d67f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 6 Jun 2024 12:14:42 +0200 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=96=8B=EF=B8=8F=20reduce=20coupling?= =?UTF-8?q?=20+=20more=20docs=20or=20build=5Flocal=5Frun?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/build_local_run.py | 436 +++++++++++++------------------ fdroidserver/build_local_sudo.py | 2 +- fdroidserver/common.py | 55 +++- fdroidserver/metadata.py | 2 +- 4 files changed, 230 insertions(+), 265 deletions(-) diff --git a/fdroidserver/build_local_run.py b/fdroidserver/build_local_run.py index 10f7caf5..9a239e88 100644 --- a/fdroidserver/build_local_run.py +++ b/fdroidserver/build_local_run.py @@ -1,8 +1,8 @@ import os +import re import sys import glob import shutil -import gettext import logging import pathlib import tarfile @@ -16,8 +16,17 @@ import fdroidserver.exception from fdroidserver import _ -def rlimit_check(apps_count): - """make sure enough open files are allowed to process everything""" +def rlimit_check(apps_count=1): + """Make sure linux is confgured to allow for enough simultaneously oepn files. + + TODO: check if this is obsolete + + Parameters + ---------- + apps_count + In the past this used to be `len(apps)` In this context we're + always buidling just one app so this is always 1 + """ try: import resource # not available on Windows @@ -37,7 +46,20 @@ def rlimit_check(apps_count): pass -def get_ndk_path(build): +def install_ndk(build, config): + """Make sure the requested NDK version is or gets installed. + + TODO: check if this should be moved to a script that runs before starting + the build. e.g. `build_local_prepare` or `build_local_sudo` + + Parameters + ---------- + build + Metadata build entry that's about the build and may contain the + requested NDK version + config + dictonariy holding config/default data from `./config.yml` + """ ndk_path = build.ndk_path() if build.ndk or (build.buildjni and build.buildjni != ['no']): if not ndk_path: @@ -47,13 +69,11 @@ def get_ndk_path(build): if k.endswith("_orig"): continue logging.warning(" %s: %s" % (k, v)) - if onserver: - fdroidserver.common.auto_install_ndk(build) - else: - raise fdroidserver.exception.FDroidException() + fdroidserver.common.auto_install_ndk(build) elif not os.path.isdir(ndk_path): logging.critical("Android NDK '%s' is not a directory!" % ndk_path) raise fdroidserver.exception.FDroidException() + return ndk_path def get_build_root_dir(app, build): @@ -71,14 +91,44 @@ def transform_first_char(string, method): return method(string[0]) + string[1:] -def init_build_subprocess(app, build, config): - # We need to clean via the build tool in case the binary dirs are - # different from the default ones +def get_flavours_cmd(build): + """Get flavor string, preformatted for gradle cli. + + Reads build flavors form metadata if any and reformats and concatenates + them to be ready for use as CLI arguments to gradle. This will treat the + vlue 'yes' as if there were not particular build flavor selected. + + Parameters + ---------- + build + The metadata build entry you'd like to read flavors from + + Returns + ------- + A string containing the build flavor for this build. If it's the default + flavor ("yes" in metadata) this returns an empty string. Returns None if + it's not a gradle build. + """ + flavours = build.gradle + + if flavours == ['yes']: + flavours = [] + + flavours_cmd = ''.join( + [transform_first_char(flav, str.upper) for flav in flavours] + ) + + return flavours_cmd + + +def init_build(app, build, config): root_dir = get_build_root_dir(app, build) p = None gradletasks = [] - flavours_cmd = None + + # We need to clean via the build tool in case the binary dirs are + # different from the default ones bmethod = build.build_method() if bmethod == 'maven': @@ -100,13 +150,7 @@ def init_build_subprocess(app, build, config): if build.preassemble: gradletasks += build.preassemble - flavours = build.gradle - if flavours == ['yes']: - flavours = [] - - flavours_cmd = ''.join( - [transform_first_char(flav, str.upper) for flav in flavours] - ) + flavours_cmd = get_flavours_cmd(build) gradletasks += ['assemble' + flavours_cmd + 'Release'] @@ -133,10 +177,22 @@ def init_build_subprocess(app, build, config): "Error cleaning %s:%s" % (app.id, build.versionName), p.output ) - return p, gradletasks, flavours_cmd + return gradletasks def sanitize_build_dir(app): + """Delete build output directories. + + This function deletes the default build/binary/target/... output + directories for follwoing build tools: gradle, maven, ant, jni. It also + deletes gradle-wrapper if present. It just uses parths, hardcoded here, + it doesn't call and build system clean routines. + + Parameters + ---------- + app + The metadata of the app to sanitize + """ build_dir = fdroidserver.common.get_build_dir(app) for root, dirs, files in os.walk(build_dir): @@ -207,8 +263,20 @@ def make_tarball(app, build, tmp_dir): def execute_build_commands(app, build): - build_dir = fdroidserver.common.get_build_dir(app) - srclibpaths = get_srclibpaths(build, build_dir) + """Execute `bulid` commands if present in metadata. + + see: https://f-droid.org/docs/Build_Metadata_Reference/#build_build + + Parameters + ---------- + app + metadata app object + build + metadata build object + """ + root_dir = get_build_root_dir(app, build) + srclibpaths = get_srclibpaths(app, build) + if build.build: logging.info("Running 'build' commands in %s" % root_dir) cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build) @@ -228,22 +296,45 @@ def execute_build_commands(app, build): ) -def get_srclibpaths(build, srclib_dir): +def get_srclibpaths(app, build): + """Get srclibpaths list of tuples. + + This will just assemble the srclibpaths list of tuples, it won't fetch + or checkout any source code, identical to return value of + common.prepare_souce(). + + Parameters + ---------- + app + metadata app object + build + metadata build object + + Returns + ------- + List of srclibpath tuples + """ + vcs, _ = fdroidserver.common.setup_vcs(app) + srclibpaths = [] if build.srclibs: logging.info("Collecting source libraries") for lib in build.srclibs: srclibpaths.append( fdroidserver.common.getsrclib( - lib, srclib_dir, preponly=onserver, refresh=refresh, build=build + lib, "./build/srclib", prepare=False, refresh=False, build=build ) ) + + basesrclib = vcs.getsrclib() + if basesrclib: + srclibpaths.append(basesrclib) + return srclibpaths -def execute_buildjni_commands(app, build): +def execute_buildjni_commands(app, build, ndk_path): root_dir = get_build_root_dir(app, build) - ndk_path = get_ndk_path(build) if build.buildjni and build.buildjni != ['no']: logging.info("Building the native code") jni_components = build.buildjni @@ -274,11 +365,11 @@ def execute_buildjni_commands(app, build): ) -def execute_build___(app, build, config, gradletasks): - build_dir = fdroidserver.common.get_build_dir(app) +def execute_build(app, build, config, gradletasks): root_dir = get_build_root_dir(app, build) p = None + bindir = None bmethod = build.build_method() if bmethod == 'maven': logging.info("Building Maven project...") @@ -344,221 +435,10 @@ def execute_build___(app, build, config, gradletasks): bindir = os.path.join(root_dir, 'bin') - if os.path.isdir(os.path.join(build_dir, '.git')): - import git - - commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) - else: - commit_id = build.commit - - if p is not None and p.returncode != 0: - raise fdroidserver.exception.BuildException( - "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), - p.output, - ) - logging.info( - "Successfully built version {versionName} of {appid} from {commit_id}".format( - versionName=build.versionName, appid=app.id, commit_id=commit_id - ) - ) - - return p + return p, bindir -def collect_build_output(app, build, p, flavours_cmd): - root_dir = get_build_root_dir(app, build) - - omethod = build.output_method() - if omethod == 'maven': - stdout_apk = '\n'.join( - [ - line - for line in p.output.splitlines() - if any(a in line for a in ('.apk', '.ap_', '.jar')) - ] - ) - m = re.match( - r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M - ) - if not m: - m = re.match( - r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", - stdout_apk, - re.S | re.M, - ) - if not m: - m = re.match( - r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' - + bindir - + r'/([^/]+)\.ap[_k][,\]]', - stdout_apk, - re.S | re.M, - ) - - if not m: - m = re.match( - r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", - stdout_apk, - re.S | re.M, - ) - if not m: - raise fdroidserver.exception.BuildException('Failed to find output') - src = m.group(1) - src = os.path.join(bindir, src) + '.apk' - - elif omethod == 'gradle': - src = None - apk_dirs = [ - # gradle plugin >= 3.0 - os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'), - # gradle plugin < 3.0 and >= 0.11 - os.path.join(root_dir, 'build', 'outputs', 'apk'), - # really old path - os.path.join(root_dir, 'build', 'apk'), - ] - # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in - # a subdirectory corresponding to the flavour command used, but with different - # capitalization. - if flavours_cmd: - apk_dirs.append( - os.path.join( - root_dir, - 'build', - 'outputs', - 'apk', - transform_first_char(flavours_cmd, str.lower), - 'release', - ) - ) - for apks_dir in apk_dirs: - for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: - apks = glob.glob(os.path.join(apks_dir, apkglob)) - - if len(apks) > 1: - raise fdroidserver.exception.BuildException( - 'More than one resulting apks found in %s' % apks_dir, - '\n'.join(apks), - ) - if len(apks) == 1: - src = apks[0] - break - if src is not None: - break - - if src is None: - raise fdroidserver.exception.BuildException('Failed to find any output apks') - - elif omethod == 'ant': - stdout_apk = '\n'.join( - [line for line in p.output.splitlines() if '.apk' in line] - ) - src = re.match( - r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M - ).group(1) - src = os.path.join(bindir, src) - elif omethod == 'raw': - output_path = fdroidserver.common.replace_build_vars(build.output, build) - globpath = os.path.join(root_dir, output_path) - apks = glob.glob(globpath) - if len(apks) > 1: - raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) - if len(apks) < 1: - raise fdroidserver.exception.BuildException('No apks match %s' % globpath) - src = os.path.normpath(apks[0]) - - -def buildbuildbuild___(app, build, config, gradletasks): - root_dir = get_build_root_dir(app, build) - - bmethod = build.build_method() - if bmethod == 'maven': - logging.info("Building Maven project...") - - if '@' in build.maven: - maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) - else: - maven_dir = root_dir - - mvncmd = [ - config['mvn3'], - '-Dandroid.sdk.path=' + config['sdk_path'], - '-Dmaven.jar.sign.skip=true', - '-Dmaven.test.skip=true', - '-Dandroid.sign.debug=false', - '-Dandroid.release=true', - 'package', - ] - if build.target: - target = build.target.split('-')[1] - fdroidserver.common.regsub_file( - r'[0-9]*', - r'%s' % target, - os.path.join(root_dir, 'pom.xml'), - ) - if '@' in build.maven: - fdroidserver.common.regsub_file( - r'[0-9]*', - r'%s' % target, - os.path.join(maven_dir, 'pom.xml'), - ) - - p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir) - - bindir = os.path.join(root_dir, 'target') - - elif bmethod == 'gradle': - logging.info("Building Gradle project...") - - cmd = [config['gradle']] - if build.gradleprops: - cmd += ['-P' + kv for kv in build.gradleprops] - - cmd += gradletasks - - p = fdroidserver.common.FDroidPopen( - cmd, - cwd=root_dir, - envs={ - "GRADLE_VERSION_DIR": config['gradle_version_dir'], - "CACHEDIR": config['cachedir'], - }, - ) - - elif bmethod == 'ant': - logging.info("Building Ant project...") - cmd = ['ant'] - if build.antcommands: - cmd += build.antcommands - else: - cmd += ['release'] - p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir) - - bindir = os.path.join(root_dir, 'bin') - - -def check_build_success(app, build, p): - build_dir = fdroidserver.common.get_build_dir(app) - - if os.path.isdir(os.path.join(build_dir, '.git')): - import git - - commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) - else: - commit_id = build.commit - - if p is not None and p.returncode != 0: - raise fdroidserver.exception.BuildException( - "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), - p.output, - ) - logging.info( - "Successfully built version {versionName} of {appid} from {commit_id}".format( - versionName=build.versionName, appid=app.id, commit_id=commit_id - ) - ) - - -def collect_output_again(app, build, p, flavours_cmd): +def collect_build_output(app, build, p, bindir): root_dir = get_build_root_dir(app, build) omethod = build.output_method() @@ -613,6 +493,7 @@ def collect_output_again(app, build, p, flavours_cmd): # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in # a subdirectory corresponding to the flavour command used, but with different # capitalization. + flavours_cmd = get_flavours_cmd(build) if flavours_cmd: apk_dirs.append( os.path.join( @@ -640,7 +521,9 @@ def collect_output_again(app, build, p, flavours_cmd): break if src is None: - raise fdroidserver.exception.BuildException('Failed to find any output apks') + raise fdroidserver.exception.BuildException( + 'Failed to find any output apks' + ) elif omethod == 'ant': stdout_apk = '\n'.join( @@ -655,17 +538,40 @@ def collect_output_again(app, build, p, flavours_cmd): globpath = os.path.join(root_dir, output_path) apks = glob.glob(globpath) if len(apks) > 1: - raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) + raise fdroidserver.exception.BuildException( + 'Multiple apks match %s' % globpath, '\n'.join(apks) + ) if len(apks) < 1: raise fdroidserver.exception.BuildException('No apks match %s' % globpath) src = os.path.normpath(apks[0]) return src -def execute_postbuild(app, build, src): +def check_build_success(app, build, p): build_dir = fdroidserver.common.get_build_dir(app) + + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + + commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output, + ) + logging.info( + "Successfully built version {versionName} of {appid} from {commit_id}".format( + versionName=build.versionName, appid=app.id, commit_id=commit_id + ) + ) + + +def execute_postbuild(app, build, src): root_dir = get_build_root_dir(app, build) - srclibpaths = get_srclibpaths(build, build_dir) + srclibpaths = get_srclibpaths(app, build) if build.postbuild: logging.info(f"Running 'postbuild' commands in {root_dir}") @@ -673,9 +579,9 @@ def execute_postbuild(app, build, src): # Substitute source library paths into commands... for name, number, libpath in srclibpaths: - cmd = cmd.replace(f"$${name}$$", str(Path.cwd() / libpath)) + cmd = cmd.replace(f"$${name}$$", str(pathlib.Path.cwd() / libpath)) - cmd = cmd.replace('$$OUT$$', str(Path(src).resolve())) + cmd = cmd.replace('$$OUT$$', str(pathlib.Path(src).resolve())) p = fdroidserver.common.FDroidPopen( ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir @@ -722,14 +628,20 @@ def get_metadata_from_apk(app, build, apkfile): native_code = fdroidserver.common.get_native_code(apkfile) if build.buildjni and build.buildjni != ['no'] and not native_code: - raise fdroidserver.exception.BuildException("Native code should have been built but none was packaged") + raise fdroidserver.exception.BuildException( + "Native code should have been built but none was packaged" + ) if build.novcheck: versionCode = build.versionCode versionName = build.versionName if not versionCode or versionName is None: - raise fdroidserver.exception.BuildException("Could not find version information in build in output") + raise fdroidserver.exception.BuildException( + "Could not find version information in build in output" + ) if not appid: - raise fdroidserver.exception.BuildException("Could not find package ID in output") + raise fdroidserver.exception.BuildException( + "Could not find package ID in output" + ) if appid != app.id: raise fdroidserver.exception.BuildException( "Wrong package ID - build " + appid + " but expected " + app.id @@ -749,7 +661,9 @@ def validate_build_artifacts(app, build, src): # code in our new APK match what we expect... logging.debug("Checking " + src) if not os.path.exists(src): - raise fdroidserver.exception.BuildException("Unsigned APK is not at expected location of " + src) + raise fdroidserver.exception.BuildException( + "Unsigned APK is not at expected location of " + src + ) if fdroidserver.common.get_file_extension(src) == 'apk': vercode, version = get_metadata_from_apk(app, build, src) @@ -783,16 +697,15 @@ def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball def run_this_build(config, options, package_name, version_code): - """run build for one specific version of an app localy + """Run build for one specific version of an app localy. :raises: various exceptions in case and of the pre-required conditions for the requested build are not met """ - app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) # not sure if this makes any sense to change open file limits since we know # that this script will only ever build one app - rlimit_check(1) + rlimit_check() logging.info( "Building version %s (%s) of %s" @@ -801,7 +714,7 @@ def run_this_build(config, options, package_name, version_code): # init fdroid Popen wrapper fdroidserver.common.set_FDroidPopen_env(build) - p, gradletasks, flavours_cmd = init_build_subprocess(app, build, config) + gradletasks = init_build(app, build, config) sanitize_build_dir(app) @@ -817,14 +730,13 @@ def run_this_build(config, options, package_name, version_code): execute_build_commands(app, build) # Build native stuff if required... - execute_buildjni_commands(app, build) + ndk_path = install_ndk(build, config) # TODO: move to prepare step? + execute_buildjni_commands(app, build, ndk_path) # Build the release... - p = execute_build___(app, build, config, gradletasks) - collect_build_output(app, build, p, flavours_cmd) - buildbuildbuild___(app, build, config, gradletasks) + p, bindir = execute_build(app, build, config, gradletasks) check_build_success(app, build, p) - src = collect_output_again(app, build, p, flavours_cmd) + src = collect_build_output(app, build, p, bindir) # Run a postbuild command if one is required... execute_postbuild(app, build, src) @@ -835,6 +747,7 @@ def run_this_build(config, options, package_name, version_code): move_build_output(app, build, src, tmp_dir) + def main(): parser = argparse.ArgumentParser( description=_( @@ -844,7 +757,8 @@ def main(): ) ) parser.add_argument( - "APP_VERSION", help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)") + "APP_VERSION", + help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)"), ) # fdroid args/opts boilerplate diff --git a/fdroidserver/build_local_sudo.py b/fdroidserver/build_local_sudo.py index e878edae..c78c1908 100644 --- a/fdroidserver/build_local_sudo.py +++ b/fdroidserver/build_local_sudo.py @@ -90,7 +90,6 @@ def main(): package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION) app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) - # intialize FDroidPopen # TODO: remove once FDroidPopen is replaced with vm/container exec fdroidserver.common.set_FDroidPopen_env(build) @@ -100,5 +99,6 @@ def main(): sudo_lock_root(app, build) sudo_uninstall(app, build) + if __name__ == "__main__": main() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 52e65aa4..735afae1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1148,7 +1148,24 @@ def get_head_commit_id(git_repo): def setup_vcs(app): - """Checkout code from VCS and return instance of vcs and the build dir.""" + """Create a VCS instance for given app. + + This is a factory function that creates the correct type of VCS instance. + This doesn't checkout or clone any source code, it just creates a VCS + instance. + + Parameters + ---------- + app + metadata app object + + Returns + ------- + vcs + VCS instance corresponding to passed app + build_dir + source code checkout directory for the supplied app + """ build_dir = get_build_dir(app) # Set up vcs interface and make sure we have the latest code... @@ -2213,7 +2230,41 @@ def getsrclib(spec, srclib_dir, basepath=False, referencing it, which may be a subdirectory of the actual project. If you want the base directory of the project, pass 'basepath=True'. - spec and srclib_dir are both strings, not pathlib.Path. + Parameters + ---------- + spec + srclib identifier (e.g. 'reproducible-apk-tools@v0.2.3'). + must be string. + srclib_dir + base dir for holding checkouts of srclibs (usually './build/srclib'). + must be a string. + basepath + changes the output of libdir to the base path, if set to True (default: + False) + raw + Don't sparese the spec instead use the unparsed spec as name, if set to + True (default: False) + prepare + Don't run `Prepare` commands in metadata, if set to False (default: + True) + preponly + Don't checkout the latest source code, if set to True (default: False) + refresh + Don't fetch latest source code from git remote, if set to False + (default: True) + build + metadata build object + + Returns + ------- + name + name of the srclib (e.g. 'mylib' when the spec is 'mylib@1.2.3') + number + number prefix from srclib spec (e.g. '7' when spec is '7:mylib@1.2.3') + (only used for ant builds) + libdir + (sub-)directory with the source code of this srclib (if basepath is set + this will ignore 'Subdir' from srclib metadata) """ number = None subdir = None diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 74a663c1..63e26994 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -608,7 +608,7 @@ def read_metadata(appids={}, sort_by_time=False): def read_build_metadata(package_name, version_code, check_disabled=True): - """read 1 single metadata file from the file system + 1 singled out build + """Read 1 single metadata file from the file system + 1 singled out build. Parameters ----------