From 629fd1a204ec6efb2b4e9ff2f9859f68c4d67f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 6 Jun 2024 12:14:42 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=96=8B=EF=B8=8F=20reduce=20coupling=20+?= =?UTF-8?q?=20more=20docs=20or=20build=5Flocal=5Frun?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fdroidserver/build_local_run.py | 436 +++++++++++++------------------ fdroidserver/build_local_sudo.py | 2 +- fdroidserver/common.py | 55 +++- fdroidserver/metadata.py | 2 +- 4 files changed, 230 insertions(+), 265 deletions(-) diff --git a/fdroidserver/build_local_run.py b/fdroidserver/build_local_run.py index 10f7caf5..9a239e88 100644 --- a/fdroidserver/build_local_run.py +++ b/fdroidserver/build_local_run.py @@ -1,8 +1,8 @@ import os +import re import sys import glob import shutil -import gettext import logging import pathlib import tarfile @@ -16,8 +16,17 @@ import fdroidserver.exception from fdroidserver import _ -def rlimit_check(apps_count): - """make sure enough open files are allowed to process everything""" +def rlimit_check(apps_count=1): + """Make sure linux is confgured to allow for enough simultaneously oepn files. + + TODO: check if this is obsolete + + Parameters + ---------- + apps_count + In the past this used to be `len(apps)` In this context we're + always buidling just one app so this is always 1 + """ try: import resource # not available on Windows @@ -37,7 +46,20 @@ def rlimit_check(apps_count): pass -def get_ndk_path(build): +def install_ndk(build, config): + """Make sure the requested NDK version is or gets installed. + + TODO: check if this should be moved to a script that runs before starting + the build. e.g. `build_local_prepare` or `build_local_sudo` + + Parameters + ---------- + build + Metadata build entry that's about the build and may contain the + requested NDK version + config + dictonariy holding config/default data from `./config.yml` + """ ndk_path = build.ndk_path() if build.ndk or (build.buildjni and build.buildjni != ['no']): if not ndk_path: @@ -47,13 +69,11 @@ def get_ndk_path(build): if k.endswith("_orig"): continue logging.warning(" %s: %s" % (k, v)) - if onserver: - fdroidserver.common.auto_install_ndk(build) - else: - raise fdroidserver.exception.FDroidException() + fdroidserver.common.auto_install_ndk(build) elif not os.path.isdir(ndk_path): logging.critical("Android NDK '%s' is not a directory!" % ndk_path) raise fdroidserver.exception.FDroidException() + return ndk_path def get_build_root_dir(app, build): @@ -71,14 +91,44 @@ def transform_first_char(string, method): return method(string[0]) + string[1:] -def init_build_subprocess(app, build, config): - # We need to clean via the build tool in case the binary dirs are - # different from the default ones +def get_flavours_cmd(build): + """Get flavor string, preformatted for gradle cli. + + Reads build flavors form metadata if any and reformats and concatenates + them to be ready for use as CLI arguments to gradle. This will treat the + vlue 'yes' as if there were not particular build flavor selected. + + Parameters + ---------- + build + The metadata build entry you'd like to read flavors from + + Returns + ------- + A string containing the build flavor for this build. If it's the default + flavor ("yes" in metadata) this returns an empty string. Returns None if + it's not a gradle build. + """ + flavours = build.gradle + + if flavours == ['yes']: + flavours = [] + + flavours_cmd = ''.join( + [transform_first_char(flav, str.upper) for flav in flavours] + ) + + return flavours_cmd + + +def init_build(app, build, config): root_dir = get_build_root_dir(app, build) p = None gradletasks = [] - flavours_cmd = None + + # We need to clean via the build tool in case the binary dirs are + # different from the default ones bmethod = build.build_method() if bmethod == 'maven': @@ -100,13 +150,7 @@ def init_build_subprocess(app, build, config): if build.preassemble: gradletasks += build.preassemble - flavours = build.gradle - if flavours == ['yes']: - flavours = [] - - flavours_cmd = ''.join( - [transform_first_char(flav, str.upper) for flav in flavours] - ) + flavours_cmd = get_flavours_cmd(build) gradletasks += ['assemble' + flavours_cmd + 'Release'] @@ -133,10 +177,22 @@ def init_build_subprocess(app, build, config): "Error cleaning %s:%s" % (app.id, build.versionName), p.output ) - return p, gradletasks, flavours_cmd + return gradletasks def sanitize_build_dir(app): + """Delete build output directories. + + This function deletes the default build/binary/target/... output + directories for follwoing build tools: gradle, maven, ant, jni. It also + deletes gradle-wrapper if present. It just uses parths, hardcoded here, + it doesn't call and build system clean routines. + + Parameters + ---------- + app + The metadata of the app to sanitize + """ build_dir = fdroidserver.common.get_build_dir(app) for root, dirs, files in os.walk(build_dir): @@ -207,8 +263,20 @@ def make_tarball(app, build, tmp_dir): def execute_build_commands(app, build): - build_dir = fdroidserver.common.get_build_dir(app) - srclibpaths = get_srclibpaths(build, build_dir) + """Execute `bulid` commands if present in metadata. + + see: https://f-droid.org/docs/Build_Metadata_Reference/#build_build + + Parameters + ---------- + app + metadata app object + build + metadata build object + """ + root_dir = get_build_root_dir(app, build) + srclibpaths = get_srclibpaths(app, build) + if build.build: logging.info("Running 'build' commands in %s" % root_dir) cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build) @@ -228,22 +296,45 @@ def execute_build_commands(app, build): ) -def get_srclibpaths(build, srclib_dir): +def get_srclibpaths(app, build): + """Get srclibpaths list of tuples. + + This will just assemble the srclibpaths list of tuples, it won't fetch + or checkout any source code, identical to return value of + common.prepare_souce(). + + Parameters + ---------- + app + metadata app object + build + metadata build object + + Returns + ------- + List of srclibpath tuples + """ + vcs, _ = fdroidserver.common.setup_vcs(app) + srclibpaths = [] if build.srclibs: logging.info("Collecting source libraries") for lib in build.srclibs: srclibpaths.append( fdroidserver.common.getsrclib( - lib, srclib_dir, preponly=onserver, refresh=refresh, build=build + lib, "./build/srclib", prepare=False, refresh=False, build=build ) ) + + basesrclib = vcs.getsrclib() + if basesrclib: + srclibpaths.append(basesrclib) + return srclibpaths -def execute_buildjni_commands(app, build): +def execute_buildjni_commands(app, build, ndk_path): root_dir = get_build_root_dir(app, build) - ndk_path = get_ndk_path(build) if build.buildjni and build.buildjni != ['no']: logging.info("Building the native code") jni_components = build.buildjni @@ -274,11 +365,11 @@ def execute_buildjni_commands(app, build): ) -def execute_build___(app, build, config, gradletasks): - build_dir = fdroidserver.common.get_build_dir(app) +def execute_build(app, build, config, gradletasks): root_dir = get_build_root_dir(app, build) p = None + bindir = None bmethod = build.build_method() if bmethod == 'maven': logging.info("Building Maven project...") @@ -344,221 +435,10 @@ def execute_build___(app, build, config, gradletasks): bindir = os.path.join(root_dir, 'bin') - if os.path.isdir(os.path.join(build_dir, '.git')): - import git - - commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) - else: - commit_id = build.commit - - if p is not None and p.returncode != 0: - raise fdroidserver.exception.BuildException( - "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), - p.output, - ) - logging.info( - "Successfully built version {versionName} of {appid} from {commit_id}".format( - versionName=build.versionName, appid=app.id, commit_id=commit_id - ) - ) - - return p + return p, bindir -def collect_build_output(app, build, p, flavours_cmd): - root_dir = get_build_root_dir(app, build) - - omethod = build.output_method() - if omethod == 'maven': - stdout_apk = '\n'.join( - [ - line - for line in p.output.splitlines() - if any(a in line for a in ('.apk', '.ap_', '.jar')) - ] - ) - m = re.match( - r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M - ) - if not m: - m = re.match( - r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", - stdout_apk, - re.S | re.M, - ) - if not m: - m = re.match( - r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' - + bindir - + r'/([^/]+)\.ap[_k][,\]]', - stdout_apk, - re.S | re.M, - ) - - if not m: - m = re.match( - r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", - stdout_apk, - re.S | re.M, - ) - if not m: - raise fdroidserver.exception.BuildException('Failed to find output') - src = m.group(1) - src = os.path.join(bindir, src) + '.apk' - - elif omethod == 'gradle': - src = None - apk_dirs = [ - # gradle plugin >= 3.0 - os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'), - # gradle plugin < 3.0 and >= 0.11 - os.path.join(root_dir, 'build', 'outputs', 'apk'), - # really old path - os.path.join(root_dir, 'build', 'apk'), - ] - # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in - # a subdirectory corresponding to the flavour command used, but with different - # capitalization. - if flavours_cmd: - apk_dirs.append( - os.path.join( - root_dir, - 'build', - 'outputs', - 'apk', - transform_first_char(flavours_cmd, str.lower), - 'release', - ) - ) - for apks_dir in apk_dirs: - for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: - apks = glob.glob(os.path.join(apks_dir, apkglob)) - - if len(apks) > 1: - raise fdroidserver.exception.BuildException( - 'More than one resulting apks found in %s' % apks_dir, - '\n'.join(apks), - ) - if len(apks) == 1: - src = apks[0] - break - if src is not None: - break - - if src is None: - raise fdroidserver.exception.BuildException('Failed to find any output apks') - - elif omethod == 'ant': - stdout_apk = '\n'.join( - [line for line in p.output.splitlines() if '.apk' in line] - ) - src = re.match( - r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M - ).group(1) - src = os.path.join(bindir, src) - elif omethod == 'raw': - output_path = fdroidserver.common.replace_build_vars(build.output, build) - globpath = os.path.join(root_dir, output_path) - apks = glob.glob(globpath) - if len(apks) > 1: - raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) - if len(apks) < 1: - raise fdroidserver.exception.BuildException('No apks match %s' % globpath) - src = os.path.normpath(apks[0]) - - -def buildbuildbuild___(app, build, config, gradletasks): - root_dir = get_build_root_dir(app, build) - - bmethod = build.build_method() - if bmethod == 'maven': - logging.info("Building Maven project...") - - if '@' in build.maven: - maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) - else: - maven_dir = root_dir - - mvncmd = [ - config['mvn3'], - '-Dandroid.sdk.path=' + config['sdk_path'], - '-Dmaven.jar.sign.skip=true', - '-Dmaven.test.skip=true', - '-Dandroid.sign.debug=false', - '-Dandroid.release=true', - 'package', - ] - if build.target: - target = build.target.split('-')[1] - fdroidserver.common.regsub_file( - r'[0-9]*', - r'%s' % target, - os.path.join(root_dir, 'pom.xml'), - ) - if '@' in build.maven: - fdroidserver.common.regsub_file( - r'[0-9]*', - r'%s' % target, - os.path.join(maven_dir, 'pom.xml'), - ) - - p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir) - - bindir = os.path.join(root_dir, 'target') - - elif bmethod == 'gradle': - logging.info("Building Gradle project...") - - cmd = [config['gradle']] - if build.gradleprops: - cmd += ['-P' + kv for kv in build.gradleprops] - - cmd += gradletasks - - p = fdroidserver.common.FDroidPopen( - cmd, - cwd=root_dir, - envs={ - "GRADLE_VERSION_DIR": config['gradle_version_dir'], - "CACHEDIR": config['cachedir'], - }, - ) - - elif bmethod == 'ant': - logging.info("Building Ant project...") - cmd = ['ant'] - if build.antcommands: - cmd += build.antcommands - else: - cmd += ['release'] - p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir) - - bindir = os.path.join(root_dir, 'bin') - - -def check_build_success(app, build, p): - build_dir = fdroidserver.common.get_build_dir(app) - - if os.path.isdir(os.path.join(build_dir, '.git')): - import git - - commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) - else: - commit_id = build.commit - - if p is not None and p.returncode != 0: - raise fdroidserver.exception.BuildException( - "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), - p.output, - ) - logging.info( - "Successfully built version {versionName} of {appid} from {commit_id}".format( - versionName=build.versionName, appid=app.id, commit_id=commit_id - ) - ) - - -def collect_output_again(app, build, p, flavours_cmd): +def collect_build_output(app, build, p, bindir): root_dir = get_build_root_dir(app, build) omethod = build.output_method() @@ -613,6 +493,7 @@ def collect_output_again(app, build, p, flavours_cmd): # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in # a subdirectory corresponding to the flavour command used, but with different # capitalization. + flavours_cmd = get_flavours_cmd(build) if flavours_cmd: apk_dirs.append( os.path.join( @@ -640,7 +521,9 @@ def collect_output_again(app, build, p, flavours_cmd): break if src is None: - raise fdroidserver.exception.BuildException('Failed to find any output apks') + raise fdroidserver.exception.BuildException( + 'Failed to find any output apks' + ) elif omethod == 'ant': stdout_apk = '\n'.join( @@ -655,17 +538,40 @@ def collect_output_again(app, build, p, flavours_cmd): globpath = os.path.join(root_dir, output_path) apks = glob.glob(globpath) if len(apks) > 1: - raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) + raise fdroidserver.exception.BuildException( + 'Multiple apks match %s' % globpath, '\n'.join(apks) + ) if len(apks) < 1: raise fdroidserver.exception.BuildException('No apks match %s' % globpath) src = os.path.normpath(apks[0]) return src -def execute_postbuild(app, build, src): +def check_build_success(app, build, p): build_dir = fdroidserver.common.get_build_dir(app) + + if os.path.isdir(os.path.join(build_dir, '.git')): + import git + + commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir)) + else: + commit_id = build.commit + + if p is not None and p.returncode != 0: + raise fdroidserver.exception.BuildException( + "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id), + p.output, + ) + logging.info( + "Successfully built version {versionName} of {appid} from {commit_id}".format( + versionName=build.versionName, appid=app.id, commit_id=commit_id + ) + ) + + +def execute_postbuild(app, build, src): root_dir = get_build_root_dir(app, build) - srclibpaths = get_srclibpaths(build, build_dir) + srclibpaths = get_srclibpaths(app, build) if build.postbuild: logging.info(f"Running 'postbuild' commands in {root_dir}") @@ -673,9 +579,9 @@ def execute_postbuild(app, build, src): # Substitute source library paths into commands... for name, number, libpath in srclibpaths: - cmd = cmd.replace(f"$${name}$$", str(Path.cwd() / libpath)) + cmd = cmd.replace(f"$${name}$$", str(pathlib.Path.cwd() / libpath)) - cmd = cmd.replace('$$OUT$$', str(Path(src).resolve())) + cmd = cmd.replace('$$OUT$$', str(pathlib.Path(src).resolve())) p = fdroidserver.common.FDroidPopen( ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir @@ -722,14 +628,20 @@ def get_metadata_from_apk(app, build, apkfile): native_code = fdroidserver.common.get_native_code(apkfile) if build.buildjni and build.buildjni != ['no'] and not native_code: - raise fdroidserver.exception.BuildException("Native code should have been built but none was packaged") + raise fdroidserver.exception.BuildException( + "Native code should have been built but none was packaged" + ) if build.novcheck: versionCode = build.versionCode versionName = build.versionName if not versionCode or versionName is None: - raise fdroidserver.exception.BuildException("Could not find version information in build in output") + raise fdroidserver.exception.BuildException( + "Could not find version information in build in output" + ) if not appid: - raise fdroidserver.exception.BuildException("Could not find package ID in output") + raise fdroidserver.exception.BuildException( + "Could not find package ID in output" + ) if appid != app.id: raise fdroidserver.exception.BuildException( "Wrong package ID - build " + appid + " but expected " + app.id @@ -749,7 +661,9 @@ def validate_build_artifacts(app, build, src): # code in our new APK match what we expect... logging.debug("Checking " + src) if not os.path.exists(src): - raise fdroidserver.exception.BuildException("Unsigned APK is not at expected location of " + src) + raise fdroidserver.exception.BuildException( + "Unsigned APK is not at expected location of " + src + ) if fdroidserver.common.get_file_extension(src) == 'apk': vercode, version = get_metadata_from_apk(app, build, src) @@ -783,16 +697,15 @@ def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball def run_this_build(config, options, package_name, version_code): - """run build for one specific version of an app localy + """Run build for one specific version of an app localy. :raises: various exceptions in case and of the pre-required conditions for the requested build are not met """ - app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) # not sure if this makes any sense to change open file limits since we know # that this script will only ever build one app - rlimit_check(1) + rlimit_check() logging.info( "Building version %s (%s) of %s" @@ -801,7 +714,7 @@ def run_this_build(config, options, package_name, version_code): # init fdroid Popen wrapper fdroidserver.common.set_FDroidPopen_env(build) - p, gradletasks, flavours_cmd = init_build_subprocess(app, build, config) + gradletasks = init_build(app, build, config) sanitize_build_dir(app) @@ -817,14 +730,13 @@ def run_this_build(config, options, package_name, version_code): execute_build_commands(app, build) # Build native stuff if required... - execute_buildjni_commands(app, build) + ndk_path = install_ndk(build, config) # TODO: move to prepare step? + execute_buildjni_commands(app, build, ndk_path) # Build the release... - p = execute_build___(app, build, config, gradletasks) - collect_build_output(app, build, p, flavours_cmd) - buildbuildbuild___(app, build, config, gradletasks) + p, bindir = execute_build(app, build, config, gradletasks) check_build_success(app, build, p) - src = collect_output_again(app, build, p, flavours_cmd) + src = collect_build_output(app, build, p, bindir) # Run a postbuild command if one is required... execute_postbuild(app, build, src) @@ -835,6 +747,7 @@ def run_this_build(config, options, package_name, version_code): move_build_output(app, build, src, tmp_dir) + def main(): parser = argparse.ArgumentParser( description=_( @@ -844,7 +757,8 @@ def main(): ) ) parser.add_argument( - "APP_VERSION", help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)") + "APP_VERSION", + help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)"), ) # fdroid args/opts boilerplate diff --git a/fdroidserver/build_local_sudo.py b/fdroidserver/build_local_sudo.py index e878edae..c78c1908 100644 --- a/fdroidserver/build_local_sudo.py +++ b/fdroidserver/build_local_sudo.py @@ -90,7 +90,6 @@ def main(): package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION) app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code) - # intialize FDroidPopen # TODO: remove once FDroidPopen is replaced with vm/container exec fdroidserver.common.set_FDroidPopen_env(build) @@ -100,5 +99,6 @@ def main(): sudo_lock_root(app, build) sudo_uninstall(app, build) + if __name__ == "__main__": main() diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 52e65aa4..735afae1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1148,7 +1148,24 @@ def get_head_commit_id(git_repo): def setup_vcs(app): - """Checkout code from VCS and return instance of vcs and the build dir.""" + """Create a VCS instance for given app. + + This is a factory function that creates the correct type of VCS instance. + This doesn't checkout or clone any source code, it just creates a VCS + instance. + + Parameters + ---------- + app + metadata app object + + Returns + ------- + vcs + VCS instance corresponding to passed app + build_dir + source code checkout directory for the supplied app + """ build_dir = get_build_dir(app) # Set up vcs interface and make sure we have the latest code... @@ -2213,7 +2230,41 @@ def getsrclib(spec, srclib_dir, basepath=False, referencing it, which may be a subdirectory of the actual project. If you want the base directory of the project, pass 'basepath=True'. - spec and srclib_dir are both strings, not pathlib.Path. + Parameters + ---------- + spec + srclib identifier (e.g. 'reproducible-apk-tools@v0.2.3'). + must be string. + srclib_dir + base dir for holding checkouts of srclibs (usually './build/srclib'). + must be a string. + basepath + changes the output of libdir to the base path, if set to True (default: + False) + raw + Don't sparese the spec instead use the unparsed spec as name, if set to + True (default: False) + prepare + Don't run `Prepare` commands in metadata, if set to False (default: + True) + preponly + Don't checkout the latest source code, if set to True (default: False) + refresh + Don't fetch latest source code from git remote, if set to False + (default: True) + build + metadata build object + + Returns + ------- + name + name of the srclib (e.g. 'mylib' when the spec is 'mylib@1.2.3') + number + number prefix from srclib spec (e.g. '7' when spec is '7:mylib@1.2.3') + (only used for ant builds) + libdir + (sub-)directory with the source code of this srclib (if basepath is set + this will ignore 'Subdir' from srclib metadata) """ number = None subdir = None diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 74a663c1..63e26994 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -608,7 +608,7 @@ def read_metadata(appids={}, sort_by_time=False): def read_build_metadata(package_name, version_code, check_disabled=True): - """read 1 single metadata file from the file system + 1 singled out build + """Read 1 single metadata file from the file system + 1 singled out build. Parameters ----------