diff --git a/fdroidserver/build_local_run.py b/fdroidserver/build_local_run.py
index 10f7caf5..9a239e88 100644
--- a/fdroidserver/build_local_run.py
+++ b/fdroidserver/build_local_run.py
@@ -1,8 +1,8 @@
import os
+import re
import sys
import glob
import shutil
-import gettext
import logging
import pathlib
import tarfile
@@ -16,8 +16,17 @@ import fdroidserver.exception
from fdroidserver import _
-def rlimit_check(apps_count):
- """make sure enough open files are allowed to process everything"""
+def rlimit_check(apps_count=1):
+ """Make sure linux is confgured to allow for enough simultaneously oepn files.
+
+ TODO: check if this is obsolete
+
+ Parameters
+ ----------
+ apps_count
+ In the past this used to be `len(apps)` In this context we're
+ always buidling just one app so this is always 1
+ """
try:
import resource # not available on Windows
@@ -37,7 +46,20 @@ def rlimit_check(apps_count):
pass
-def get_ndk_path(build):
+def install_ndk(build, config):
+ """Make sure the requested NDK version is or gets installed.
+
+ TODO: check if this should be moved to a script that runs before starting
+ the build. e.g. `build_local_prepare` or `build_local_sudo`
+
+ Parameters
+ ----------
+ build
+ Metadata build entry that's about the build and may contain the
+ requested NDK version
+ config
+ dictonariy holding config/default data from `./config.yml`
+ """
ndk_path = build.ndk_path()
if build.ndk or (build.buildjni and build.buildjni != ['no']):
if not ndk_path:
@@ -47,13 +69,11 @@ def get_ndk_path(build):
if k.endswith("_orig"):
continue
logging.warning(" %s: %s" % (k, v))
- if onserver:
- fdroidserver.common.auto_install_ndk(build)
- else:
- raise fdroidserver.exception.FDroidException()
+ fdroidserver.common.auto_install_ndk(build)
elif not os.path.isdir(ndk_path):
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
raise fdroidserver.exception.FDroidException()
+ return ndk_path
def get_build_root_dir(app, build):
@@ -71,14 +91,44 @@ def transform_first_char(string, method):
return method(string[0]) + string[1:]
-def init_build_subprocess(app, build, config):
- # We need to clean via the build tool in case the binary dirs are
- # different from the default ones
+def get_flavours_cmd(build):
+ """Get flavor string, preformatted for gradle cli.
+
+ Reads build flavors form metadata if any and reformats and concatenates
+ them to be ready for use as CLI arguments to gradle. This will treat the
+ vlue 'yes' as if there were not particular build flavor selected.
+
+ Parameters
+ ----------
+ build
+ The metadata build entry you'd like to read flavors from
+
+ Returns
+ -------
+ A string containing the build flavor for this build. If it's the default
+ flavor ("yes" in metadata) this returns an empty string. Returns None if
+ it's not a gradle build.
+ """
+ flavours = build.gradle
+
+ if flavours == ['yes']:
+ flavours = []
+
+ flavours_cmd = ''.join(
+ [transform_first_char(flav, str.upper) for flav in flavours]
+ )
+
+ return flavours_cmd
+
+
+def init_build(app, build, config):
root_dir = get_build_root_dir(app, build)
p = None
gradletasks = []
- flavours_cmd = None
+
+ # We need to clean via the build tool in case the binary dirs are
+ # different from the default ones
bmethod = build.build_method()
if bmethod == 'maven':
@@ -100,13 +150,7 @@ def init_build_subprocess(app, build, config):
if build.preassemble:
gradletasks += build.preassemble
- flavours = build.gradle
- if flavours == ['yes']:
- flavours = []
-
- flavours_cmd = ''.join(
- [transform_first_char(flav, str.upper) for flav in flavours]
- )
+ flavours_cmd = get_flavours_cmd(build)
gradletasks += ['assemble' + flavours_cmd + 'Release']
@@ -133,10 +177,22 @@ def init_build_subprocess(app, build, config):
"Error cleaning %s:%s" % (app.id, build.versionName), p.output
)
- return p, gradletasks, flavours_cmd
+ return gradletasks
def sanitize_build_dir(app):
+ """Delete build output directories.
+
+ This function deletes the default build/binary/target/... output
+ directories for follwoing build tools: gradle, maven, ant, jni. It also
+ deletes gradle-wrapper if present. It just uses parths, hardcoded here,
+ it doesn't call and build system clean routines.
+
+ Parameters
+ ----------
+ app
+ The metadata of the app to sanitize
+ """
build_dir = fdroidserver.common.get_build_dir(app)
for root, dirs, files in os.walk(build_dir):
@@ -207,8 +263,20 @@ def make_tarball(app, build, tmp_dir):
def execute_build_commands(app, build):
- build_dir = fdroidserver.common.get_build_dir(app)
- srclibpaths = get_srclibpaths(build, build_dir)
+ """Execute `bulid` commands if present in metadata.
+
+ see: https://f-droid.org/docs/Build_Metadata_Reference/#build_build
+
+ Parameters
+ ----------
+ app
+ metadata app object
+ build
+ metadata build object
+ """
+ root_dir = get_build_root_dir(app, build)
+ srclibpaths = get_srclibpaths(app, build)
+
if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build)
@@ -228,22 +296,45 @@ def execute_build_commands(app, build):
)
-def get_srclibpaths(build, srclib_dir):
+def get_srclibpaths(app, build):
+ """Get srclibpaths list of tuples.
+
+ This will just assemble the srclibpaths list of tuples, it won't fetch
+ or checkout any source code, identical to return value of
+ common.prepare_souce().
+
+ Parameters
+ ----------
+ app
+ metadata app object
+ build
+ metadata build object
+
+ Returns
+ -------
+ List of srclibpath tuples
+ """
+ vcs, _ = fdroidserver.common.setup_vcs(app)
+
srclibpaths = []
if build.srclibs:
logging.info("Collecting source libraries")
for lib in build.srclibs:
srclibpaths.append(
fdroidserver.common.getsrclib(
- lib, srclib_dir, preponly=onserver, refresh=refresh, build=build
+ lib, "./build/srclib", prepare=False, refresh=False, build=build
)
)
+
+ basesrclib = vcs.getsrclib()
+ if basesrclib:
+ srclibpaths.append(basesrclib)
+
return srclibpaths
-def execute_buildjni_commands(app, build):
+def execute_buildjni_commands(app, build, ndk_path):
root_dir = get_build_root_dir(app, build)
- ndk_path = get_ndk_path(build)
if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
jni_components = build.buildjni
@@ -274,11 +365,11 @@ def execute_buildjni_commands(app, build):
)
-def execute_build___(app, build, config, gradletasks):
- build_dir = fdroidserver.common.get_build_dir(app)
+def execute_build(app, build, config, gradletasks):
root_dir = get_build_root_dir(app, build)
p = None
+ bindir = None
bmethod = build.build_method()
if bmethod == 'maven':
logging.info("Building Maven project...")
@@ -344,221 +435,10 @@ def execute_build___(app, build, config, gradletasks):
bindir = os.path.join(root_dir, 'bin')
- if os.path.isdir(os.path.join(build_dir, '.git')):
- import git
-
- commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
- else:
- commit_id = build.commit
-
- if p is not None and p.returncode != 0:
- raise fdroidserver.exception.BuildException(
- "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
- p.output,
- )
- logging.info(
- "Successfully built version {versionName} of {appid} from {commit_id}".format(
- versionName=build.versionName, appid=app.id, commit_id=commit_id
- )
- )
-
- return p
+ return p, bindir
-def collect_build_output(app, build, p, flavours_cmd):
- root_dir = get_build_root_dir(app, build)
-
- omethod = build.output_method()
- if omethod == 'maven':
- stdout_apk = '\n'.join(
- [
- line
- for line in p.output.splitlines()
- if any(a in line for a in ('.apk', '.ap_', '.jar'))
- ]
- )
- m = re.match(
- r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M
- )
- if not m:
- m = re.match(
- r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
- stdout_apk,
- re.S | re.M,
- )
- if not m:
- m = re.match(
- r'.*^\[INFO\] [^$]*aapt \[package,[^$]*'
- + bindir
- + r'/([^/]+)\.ap[_k][,\]]',
- stdout_apk,
- re.S | re.M,
- )
-
- if not m:
- m = re.match(
- r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
- stdout_apk,
- re.S | re.M,
- )
- if not m:
- raise fdroidserver.exception.BuildException('Failed to find output')
- src = m.group(1)
- src = os.path.join(bindir, src) + '.apk'
-
- elif omethod == 'gradle':
- src = None
- apk_dirs = [
- # gradle plugin >= 3.0
- os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'),
- # gradle plugin < 3.0 and >= 0.11
- os.path.join(root_dir, 'build', 'outputs', 'apk'),
- # really old path
- os.path.join(root_dir, 'build', 'apk'),
- ]
- # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
- # a subdirectory corresponding to the flavour command used, but with different
- # capitalization.
- if flavours_cmd:
- apk_dirs.append(
- os.path.join(
- root_dir,
- 'build',
- 'outputs',
- 'apk',
- transform_first_char(flavours_cmd, str.lower),
- 'release',
- )
- )
- for apks_dir in apk_dirs:
- for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
- apks = glob.glob(os.path.join(apks_dir, apkglob))
-
- if len(apks) > 1:
- raise fdroidserver.exception.BuildException(
- 'More than one resulting apks found in %s' % apks_dir,
- '\n'.join(apks),
- )
- if len(apks) == 1:
- src = apks[0]
- break
- if src is not None:
- break
-
- if src is None:
- raise fdroidserver.exception.BuildException('Failed to find any output apks')
-
- elif omethod == 'ant':
- stdout_apk = '\n'.join(
- [line for line in p.output.splitlines() if '.apk' in line]
- )
- src = re.match(
- r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M
- ).group(1)
- src = os.path.join(bindir, src)
- elif omethod == 'raw':
- output_path = fdroidserver.common.replace_build_vars(build.output, build)
- globpath = os.path.join(root_dir, output_path)
- apks = glob.glob(globpath)
- if len(apks) > 1:
- raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
- if len(apks) < 1:
- raise fdroidserver.exception.BuildException('No apks match %s' % globpath)
- src = os.path.normpath(apks[0])
-
-
-def buildbuildbuild___(app, build, config, gradletasks):
- root_dir = get_build_root_dir(app, build)
-
- bmethod = build.build_method()
- if bmethod == 'maven':
- logging.info("Building Maven project...")
-
- if '@' in build.maven:
- maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
- else:
- maven_dir = root_dir
-
- mvncmd = [
- config['mvn3'],
- '-Dandroid.sdk.path=' + config['sdk_path'],
- '-Dmaven.jar.sign.skip=true',
- '-Dmaven.test.skip=true',
- '-Dandroid.sign.debug=false',
- '-Dandroid.release=true',
- 'package',
- ]
- if build.target:
- target = build.target.split('-')[1]
- fdroidserver.common.regsub_file(
- r'[0-9]*',
- r'%s' % target,
- os.path.join(root_dir, 'pom.xml'),
- )
- if '@' in build.maven:
- fdroidserver.common.regsub_file(
- r'[0-9]*',
- r'%s' % target,
- os.path.join(maven_dir, 'pom.xml'),
- )
-
- p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir)
-
- bindir = os.path.join(root_dir, 'target')
-
- elif bmethod == 'gradle':
- logging.info("Building Gradle project...")
-
- cmd = [config['gradle']]
- if build.gradleprops:
- cmd += ['-P' + kv for kv in build.gradleprops]
-
- cmd += gradletasks
-
- p = fdroidserver.common.FDroidPopen(
- cmd,
- cwd=root_dir,
- envs={
- "GRADLE_VERSION_DIR": config['gradle_version_dir'],
- "CACHEDIR": config['cachedir'],
- },
- )
-
- elif bmethod == 'ant':
- logging.info("Building Ant project...")
- cmd = ['ant']
- if build.antcommands:
- cmd += build.antcommands
- else:
- cmd += ['release']
- p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir)
-
- bindir = os.path.join(root_dir, 'bin')
-
-
-def check_build_success(app, build, p):
- build_dir = fdroidserver.common.get_build_dir(app)
-
- if os.path.isdir(os.path.join(build_dir, '.git')):
- import git
-
- commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
- else:
- commit_id = build.commit
-
- if p is not None and p.returncode != 0:
- raise fdroidserver.exception.BuildException(
- "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
- p.output,
- )
- logging.info(
- "Successfully built version {versionName} of {appid} from {commit_id}".format(
- versionName=build.versionName, appid=app.id, commit_id=commit_id
- )
- )
-
-
-def collect_output_again(app, build, p, flavours_cmd):
+def collect_build_output(app, build, p, bindir):
root_dir = get_build_root_dir(app, build)
omethod = build.output_method()
@@ -613,6 +493,7 @@ def collect_output_again(app, build, p, flavours_cmd):
# If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
# a subdirectory corresponding to the flavour command used, but with different
# capitalization.
+ flavours_cmd = get_flavours_cmd(build)
if flavours_cmd:
apk_dirs.append(
os.path.join(
@@ -640,7 +521,9 @@ def collect_output_again(app, build, p, flavours_cmd):
break
if src is None:
- raise fdroidserver.exception.BuildException('Failed to find any output apks')
+ raise fdroidserver.exception.BuildException(
+ 'Failed to find any output apks'
+ )
elif omethod == 'ant':
stdout_apk = '\n'.join(
@@ -655,17 +538,40 @@ def collect_output_again(app, build, p, flavours_cmd):
globpath = os.path.join(root_dir, output_path)
apks = glob.glob(globpath)
if len(apks) > 1:
- raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
+ raise fdroidserver.exception.BuildException(
+ 'Multiple apks match %s' % globpath, '\n'.join(apks)
+ )
if len(apks) < 1:
raise fdroidserver.exception.BuildException('No apks match %s' % globpath)
src = os.path.normpath(apks[0])
return src
-def execute_postbuild(app, build, src):
+def check_build_success(app, build, p):
build_dir = fdroidserver.common.get_build_dir(app)
+
+ if os.path.isdir(os.path.join(build_dir, '.git')):
+ import git
+
+ commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
+ else:
+ commit_id = build.commit
+
+ if p is not None and p.returncode != 0:
+ raise fdroidserver.exception.BuildException(
+ "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
+ p.output,
+ )
+ logging.info(
+ "Successfully built version {versionName} of {appid} from {commit_id}".format(
+ versionName=build.versionName, appid=app.id, commit_id=commit_id
+ )
+ )
+
+
+def execute_postbuild(app, build, src):
root_dir = get_build_root_dir(app, build)
- srclibpaths = get_srclibpaths(build, build_dir)
+ srclibpaths = get_srclibpaths(app, build)
if build.postbuild:
logging.info(f"Running 'postbuild' commands in {root_dir}")
@@ -673,9 +579,9 @@ def execute_postbuild(app, build, src):
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
- cmd = cmd.replace(f"$${name}$$", str(Path.cwd() / libpath))
+ cmd = cmd.replace(f"$${name}$$", str(pathlib.Path.cwd() / libpath))
- cmd = cmd.replace('$$OUT$$', str(Path(src).resolve()))
+ cmd = cmd.replace('$$OUT$$', str(pathlib.Path(src).resolve()))
p = fdroidserver.common.FDroidPopen(
['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir
@@ -722,14 +628,20 @@ def get_metadata_from_apk(app, build, apkfile):
native_code = fdroidserver.common.get_native_code(apkfile)
if build.buildjni and build.buildjni != ['no'] and not native_code:
- raise fdroidserver.exception.BuildException("Native code should have been built but none was packaged")
+ raise fdroidserver.exception.BuildException(
+ "Native code should have been built but none was packaged"
+ )
if build.novcheck:
versionCode = build.versionCode
versionName = build.versionName
if not versionCode or versionName is None:
- raise fdroidserver.exception.BuildException("Could not find version information in build in output")
+ raise fdroidserver.exception.BuildException(
+ "Could not find version information in build in output"
+ )
if not appid:
- raise fdroidserver.exception.BuildException("Could not find package ID in output")
+ raise fdroidserver.exception.BuildException(
+ "Could not find package ID in output"
+ )
if appid != app.id:
raise fdroidserver.exception.BuildException(
"Wrong package ID - build " + appid + " but expected " + app.id
@@ -749,7 +661,9 @@ def validate_build_artifacts(app, build, src):
# code in our new APK match what we expect...
logging.debug("Checking " + src)
if not os.path.exists(src):
- raise fdroidserver.exception.BuildException("Unsigned APK is not at expected location of " + src)
+ raise fdroidserver.exception.BuildException(
+ "Unsigned APK is not at expected location of " + src
+ )
if fdroidserver.common.get_file_extension(src) == 'apk':
vercode, version = get_metadata_from_apk(app, build, src)
@@ -783,16 +697,15 @@ def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball
def run_this_build(config, options, package_name, version_code):
- """run build for one specific version of an app localy
+ """Run build for one specific version of an app localy.
:raises: various exceptions in case and of the pre-required conditions for the requested build are not met
"""
-
app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
# not sure if this makes any sense to change open file limits since we know
# that this script will only ever build one app
- rlimit_check(1)
+ rlimit_check()
logging.info(
"Building version %s (%s) of %s"
@@ -801,7 +714,7 @@ def run_this_build(config, options, package_name, version_code):
# init fdroid Popen wrapper
fdroidserver.common.set_FDroidPopen_env(build)
- p, gradletasks, flavours_cmd = init_build_subprocess(app, build, config)
+ gradletasks = init_build(app, build, config)
sanitize_build_dir(app)
@@ -817,14 +730,13 @@ def run_this_build(config, options, package_name, version_code):
execute_build_commands(app, build)
# Build native stuff if required...
- execute_buildjni_commands(app, build)
+ ndk_path = install_ndk(build, config) # TODO: move to prepare step?
+ execute_buildjni_commands(app, build, ndk_path)
# Build the release...
- p = execute_build___(app, build, config, gradletasks)
- collect_build_output(app, build, p, flavours_cmd)
- buildbuildbuild___(app, build, config, gradletasks)
+ p, bindir = execute_build(app, build, config, gradletasks)
check_build_success(app, build, p)
- src = collect_output_again(app, build, p, flavours_cmd)
+ src = collect_build_output(app, build, p, bindir)
# Run a postbuild command if one is required...
execute_postbuild(app, build, src)
@@ -835,6 +747,7 @@ def run_this_build(config, options, package_name, version_code):
move_build_output(app, build, src, tmp_dir)
+
def main():
parser = argparse.ArgumentParser(
description=_(
@@ -844,7 +757,8 @@ def main():
)
)
parser.add_argument(
- "APP_VERSION", help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)")
+ "APP_VERSION",
+ help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)"),
)
# fdroid args/opts boilerplate
diff --git a/fdroidserver/build_local_sudo.py b/fdroidserver/build_local_sudo.py
index e878edae..c78c1908 100644
--- a/fdroidserver/build_local_sudo.py
+++ b/fdroidserver/build_local_sudo.py
@@ -90,7 +90,6 @@ def main():
package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION)
app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
-
# intialize FDroidPopen
# TODO: remove once FDroidPopen is replaced with vm/container exec
fdroidserver.common.set_FDroidPopen_env(build)
@@ -100,5 +99,6 @@ def main():
sudo_lock_root(app, build)
sudo_uninstall(app, build)
+
if __name__ == "__main__":
main()
diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 52e65aa4..735afae1 100644
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -1148,7 +1148,24 @@ def get_head_commit_id(git_repo):
def setup_vcs(app):
- """Checkout code from VCS and return instance of vcs and the build dir."""
+ """Create a VCS instance for given app.
+
+ This is a factory function that creates the correct type of VCS instance.
+ This doesn't checkout or clone any source code, it just creates a VCS
+ instance.
+
+ Parameters
+ ----------
+ app
+ metadata app object
+
+ Returns
+ -------
+ vcs
+ VCS instance corresponding to passed app
+ build_dir
+ source code checkout directory for the supplied app
+ """
build_dir = get_build_dir(app)
# Set up vcs interface and make sure we have the latest code...
@@ -2213,7 +2230,41 @@ def getsrclib(spec, srclib_dir, basepath=False,
referencing it, which may be a subdirectory of the actual project. If
you want the base directory of the project, pass 'basepath=True'.
- spec and srclib_dir are both strings, not pathlib.Path.
+ Parameters
+ ----------
+ spec
+ srclib identifier (e.g. 'reproducible-apk-tools@v0.2.3').
+ must be string.
+ srclib_dir
+ base dir for holding checkouts of srclibs (usually './build/srclib').
+ must be a string.
+ basepath
+ changes the output of libdir to the base path, if set to True (default:
+ False)
+ raw
+ Don't sparese the spec instead use the unparsed spec as name, if set to
+ True (default: False)
+ prepare
+ Don't run `Prepare` commands in metadata, if set to False (default:
+ True)
+ preponly
+ Don't checkout the latest source code, if set to True (default: False)
+ refresh
+ Don't fetch latest source code from git remote, if set to False
+ (default: True)
+ build
+ metadata build object
+
+ Returns
+ -------
+ name
+ name of the srclib (e.g. 'mylib' when the spec is 'mylib@1.2.3')
+ number
+ number prefix from srclib spec (e.g. '7' when spec is '7:mylib@1.2.3')
+ (only used for ant builds)
+ libdir
+ (sub-)directory with the source code of this srclib (if basepath is set
+ this will ignore 'Subdir' from srclib metadata)
"""
number = None
subdir = None
diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
index 74a663c1..63e26994 100644
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@@ -608,7 +608,7 @@ def read_metadata(appids={}, sort_by_time=False):
def read_build_metadata(package_name, version_code, check_disabled=True):
- """read 1 single metadata file from the file system + 1 singled out build
+ """Read 1 single metadata file from the file system + 1 singled out build.
Parameters
----------