kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-10 17:30:11 +01:00
Code Activity

🖋️ reduce coupling + more docs or build_local_run

Browse Source
This commit is contained in:
Michael Pöhn 2024-06-06 12:14:42 +02:00
parent 1bde8d27b0
commit 629fd1a204
No known key found for this signature in database
GPG Key ID: 725F386C05529A5A
4 changed files with 230 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

394
fdroidserver/build_local_run.py
View File

@ -1,8 +1,8 @@
import os
import re
import sys
import glob
import shutil
import gettext
import logging
import pathlib
import tarfile
@ -16,8 +16,17 @@ import fdroidserver.exception
from fdroidserver import _
def rlimit_check(apps_count):
"""make sure enough open files are allowed to process everything"""
def rlimit_check(apps_count=1):
"""Make sure linux is confgured to allow for enough simultaneously oepn files.
TODO: check if this is obsolete
Parameters
----------
apps_count
In the past this used to be `len(apps)` In this context we're
always buidling just one app so this is always 1
"""
try:
import resource # not available on Windows
@ -37,7 +46,20 @@ def rlimit_check(apps_count):
pass
def get_ndk_path(build):
def install_ndk(build, config):
"""Make sure the requested NDK version is or gets installed.
TODO: check if this should be moved to a script that runs before starting
the build. e.g. `build_local_prepare` or `build_local_sudo`
Parameters
----------
build
Metadata build entry that's about the build and may contain the
requested NDK version
config
dictonariy holding config/default data from `./config.yml`
"""
ndk_path = build.ndk_path()
if build.ndk or (build.buildjni and build.buildjni != ['no']):
if not ndk_path:
@ -47,13 +69,11 @@ def get_ndk_path(build):
if k.endswith("_orig"):
continue
logging.warning(" %s: %s" % (k, v))
if onserver:
fdroidserver.common.auto_install_ndk(build)
else:
raise fdroidserver.exception.FDroidException()
elif not os.path.isdir(ndk_path):
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
raise fdroidserver.exception.FDroidException()
return ndk_path
def get_build_root_dir(app, build):
@ -71,14 +91,44 @@ def transform_first_char(string, method):
return method(string[0]) + string[1:]
def init_build_subprocess(app, build, config):
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
def get_flavours_cmd(build):
"""Get flavor string, preformatted for gradle cli.
Reads build flavors form metadata if any and reformats and concatenates
them to be ready for use as CLI arguments to gradle. This will treat the
vlue 'yes' as if there were not particular build flavor selected.
Parameters
----------
build
The metadata build entry you'd like to read flavors from
Returns
-------
A string containing the build flavor for this build. If it's the default
flavor ("yes" in metadata) this returns an empty string. Returns None if
it's not a gradle build.
"""
flavours = build.gradle
if flavours == ['yes']:
flavours = []
flavours_cmd = ''.join(
[transform_first_char(flav, str.upper) for flav in flavours]
)
return flavours_cmd
def init_build(app, build, config):
root_dir = get_build_root_dir(app, build)
p = None
gradletasks = []
flavours_cmd = None
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
bmethod = build.build_method()
if bmethod == 'maven':
@ -100,13 +150,7 @@ def init_build_subprocess(app, build, config):
if build.preassemble:
gradletasks += build.preassemble
flavours = build.gradle
if flavours == ['yes']:
flavours = []
flavours_cmd = ''.join(
[transform_first_char(flav, str.upper) for flav in flavours]
)
flavours_cmd = get_flavours_cmd(build)
gradletasks += ['assemble' + flavours_cmd + 'Release']
@ -133,10 +177,22 @@ def init_build_subprocess(app, build, config):
"Error cleaning %s:%s" % (app.id, build.versionName), p.output
)
return p, gradletasks, flavours_cmd
return gradletasks
def sanitize_build_dir(app):
"""Delete build output directories.
This function deletes the default build/binary/target/... output
directories for follwoing build tools: gradle, maven, ant, jni. It also
deletes gradle-wrapper if present. It just uses parths, hardcoded here,
it doesn't call and build system clean routines.
Parameters
----------
app
The metadata of the app to sanitize
"""
build_dir = fdroidserver.common.get_build_dir(app)
for root, dirs, files in os.walk(build_dir):
@ -207,8 +263,20 @@ def make_tarball(app, build, tmp_dir):
def execute_build_commands(app, build):
build_dir = fdroidserver.common.get_build_dir(app)
srclibpaths = get_srclibpaths(build, build_dir)
"""Execute `bulid` commands if present in metadata.
see: https://f-droid.org/docs/Build_Metadata_Reference/#build_build
Parameters
----------
app
metadata app object
build
metadata build object
"""
root_dir = get_build_root_dir(app, build)
srclibpaths = get_srclibpaths(app, build)
if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build)
@ -228,22 +296,45 @@ def execute_build_commands(app, build):
)
def get_srclibpaths(build, srclib_dir):
def get_srclibpaths(app, build):
"""Get srclibpaths list of tuples.
This will just assemble the srclibpaths list of tuples, it won't fetch
or checkout any source code, identical to return value of
common.prepare_souce().
Parameters
----------
app
metadata app object
build
metadata build object
Returns
-------
List of srclibpath tuples
"""
vcs, _ = fdroidserver.common.setup_vcs(app)
srclibpaths = []
if build.srclibs:
logging.info("Collecting source libraries")
for lib in build.srclibs:
srclibpaths.append(
fdroidserver.common.getsrclib(
lib, srclib_dir, preponly=onserver, refresh=refresh, build=build
lib, "./build/srclib", prepare=False, refresh=False, build=build
)
)
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
return srclibpaths
def execute_buildjni_commands(app, build):
def execute_buildjni_commands(app, build, ndk_path):
root_dir = get_build_root_dir(app, build)
ndk_path = get_ndk_path(build)
if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
jni_components = build.buildjni
@ -274,11 +365,11 @@ def execute_buildjni_commands(app, build):
)
def execute_build___(app, build, config, gradletasks):
build_dir = fdroidserver.common.get_build_dir(app)
def execute_build(app, build, config, gradletasks):
root_dir = get_build_root_dir(app, build)
p = None
bindir = None
bmethod = build.build_method()
if bmethod == 'maven':
logging.info("Building Maven project...")
@ -344,31 +435,14 @@ def execute_build___(app, build, config, gradletasks):
bindir = os.path.join(root_dir, 'bin')
if os.path.isdir(os.path.join(build_dir, '.git')):
import git
commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
else:
commit_id = build.commit
if p is not None and p.returncode != 0:
raise fdroidserver.exception.BuildException(
"Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
p.output,
)
logging.info(
"Successfully built version {versionName} of {appid} from {commit_id}".format(
versionName=build.versionName, appid=app.id, commit_id=commit_id
)
)
return p
return p, bindir
def collect_build_output(app, build, p, flavours_cmd):
def collect_build_output(app, build, p, bindir):
root_dir = get_build_root_dir(app, build)
omethod = build.output_method()
src = None
if omethod == 'maven':
stdout_apk = '\n'.join(
[
@ -419,6 +493,7 @@ def collect_build_output(app, build, p, flavours_cmd):
# If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
# a subdirectory corresponding to the flavour command used, but with different
# capitalization.
flavours_cmd = get_flavours_cmd(build)
if flavours_cmd:
apk_dirs.append(
os.path.join(
@ -446,7 +521,9 @@ def collect_build_output(app, build, p, flavours_cmd):
break
if src is None:
raise fdroidserver.exception.BuildException('Failed to find any output apks')
raise fdroidserver.exception.BuildException(
'Failed to find any output apks'
)
elif omethod == 'ant':
stdout_apk = '\n'.join(
@ -461,79 +538,13 @@ def collect_build_output(app, build, p, flavours_cmd):
globpath = os.path.join(root_dir, output_path)
apks = glob.glob(globpath)
if len(apks) > 1:
raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
raise fdroidserver.exception.BuildException(
'Multiple apks match %s' % globpath, '\n'.join(apks)
)
if len(apks) < 1:
raise fdroidserver.exception.BuildException('No apks match %s' % globpath)
src = os.path.normpath(apks[0])
def buildbuildbuild___(app, build, config, gradletasks):
root_dir = get_build_root_dir(app, build)
bmethod = build.build_method()
if bmethod == 'maven':
logging.info("Building Maven project...")
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
else:
maven_dir = root_dir
mvncmd = [
config['mvn3'],
'-Dandroid.sdk.path=' + config['sdk_path'],
'-Dmaven.jar.sign.skip=true',
'-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false',
'-Dandroid.release=true',
'package',
]
if build.target:
target = build.target.split('-')[1]
fdroidserver.common.regsub_file(
r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(root_dir, 'pom.xml'),
)
if '@' in build.maven:
fdroidserver.common.regsub_file(
r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(maven_dir, 'pom.xml'),
)
p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif bmethod == 'gradle':
logging.info("Building Gradle project...")
cmd = [config['gradle']]
if build.gradleprops:
cmd += ['-P' + kv for kv in build.gradleprops]
cmd += gradletasks
p = fdroidserver.common.FDroidPopen(
cmd,
cwd=root_dir,
envs={
"GRADLE_VERSION_DIR": config['gradle_version_dir'],
"CACHEDIR": config['cachedir'],
},
)
elif bmethod == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if build.antcommands:
cmd += build.antcommands
else:
cmd += ['release']
p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
return src
def check_build_success(app, build, p):
@ -558,114 +569,9 @@ def check_build_success(app, build, p):
)
def collect_output_again(app, build, p, flavours_cmd):
root_dir = get_build_root_dir(app, build)
omethod = build.output_method()
src = None
if omethod == 'maven':
stdout_apk = '\n'.join(
[
line
for line in p.output.splitlines()
if any(a in line for a in ('.apk', '.ap_', '.jar'))
]
)
m = re.match(
r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M
)
if not m:
m = re.match(
r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk,
re.S | re.M,
)
if not m:
m = re.match(
r'.*^\[INFO\] [^$]*aapt \[package,[^$]*'
+ bindir
+ r'/([^/]+)\.ap[_k][,\]]',
stdout_apk,
re.S | re.M,
)
if not m:
m = re.match(
r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
stdout_apk,
re.S | re.M,
)
if not m:
raise fdroidserver.exception.BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif omethod == 'gradle':
src = None
apk_dirs = [
# gradle plugin >= 3.0
os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'),
# gradle plugin < 3.0 and >= 0.11
os.path.join(root_dir, 'build', 'outputs', 'apk'),
# really old path
os.path.join(root_dir, 'build', 'apk'),
]
# If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
# a subdirectory corresponding to the flavour command used, but with different
# capitalization.
if flavours_cmd:
apk_dirs.append(
os.path.join(
root_dir,
'build',
'outputs',
'apk',
transform_first_char(flavours_cmd, str.lower),
'release',
)
)
for apks_dir in apk_dirs:
for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
apks = glob.glob(os.path.join(apks_dir, apkglob))
if len(apks) > 1:
raise fdroidserver.exception.BuildException(
'More than one resulting apks found in %s' % apks_dir,
'\n'.join(apks),
)
if len(apks) == 1:
src = apks[0]
break
if src is not None:
break
if src is None:
raise fdroidserver.exception.BuildException('Failed to find any output apks')
elif omethod == 'ant':
stdout_apk = '\n'.join(
[line for line in p.output.splitlines() if '.apk' in line]
)
src = re.match(
r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M
).group(1)
src = os.path.join(bindir, src)
elif omethod == 'raw':
output_path = fdroidserver.common.replace_build_vars(build.output, build)
globpath = os.path.join(root_dir, output_path)
apks = glob.glob(globpath)
if len(apks) > 1:
raise fdroidserver.exception.BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
if len(apks) < 1:
raise fdroidserver.exception.BuildException('No apks match %s' % globpath)
src = os.path.normpath(apks[0])
return src
def execute_postbuild(app, build, src):
build_dir = fdroidserver.common.get_build_dir(app)
root_dir = get_build_root_dir(app, build)
srclibpaths = get_srclibpaths(build, build_dir)
srclibpaths = get_srclibpaths(app, build)
if build.postbuild:
logging.info(f"Running 'postbuild' commands in {root_dir}")
@ -673,9 +579,9 @@ def execute_postbuild(app, build, src):
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
cmd = cmd.replace(f"$${name}$$", str(Path.cwd() / libpath))
cmd = cmd.replace(f"$${name}$$", str(pathlib.Path.cwd() / libpath))
cmd = cmd.replace('$$OUT$$', str(Path(src).resolve()))
cmd = cmd.replace('$$OUT$$', str(pathlib.Path(src).resolve()))
p = fdroidserver.common.FDroidPopen(
['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir
@ -722,14 +628,20 @@ def get_metadata_from_apk(app, build, apkfile):
native_code = fdroidserver.common.get_native_code(apkfile)
if build.buildjni and build.buildjni != ['no'] and not native_code:
raise fdroidserver.exception.BuildException("Native code should have been built but none was packaged")
raise fdroidserver.exception.BuildException(
"Native code should have been built but none was packaged"
)
if build.novcheck:
versionCode = build.versionCode
versionName = build.versionName
if not versionCode or versionName is None:
raise fdroidserver.exception.BuildException("Could not find version information in build in output")
raise fdroidserver.exception.BuildException(
"Could not find version information in build in output"
)
if not appid:
raise fdroidserver.exception.BuildException("Could not find package ID in output")
raise fdroidserver.exception.BuildException(
"Could not find package ID in output"
)
if appid != app.id:
raise fdroidserver.exception.BuildException(
"Wrong package ID - build " + appid + " but expected " + app.id
@ -749,7 +661,9 @@ def validate_build_artifacts(app, build, src):
# code in our new APK match what we expect...
logging.debug("Checking " + src)
if not os.path.exists(src):
raise fdroidserver.exception.BuildException("Unsigned APK is not at expected location of " + src)
raise fdroidserver.exception.BuildException(
"Unsigned APK is not at expected location of " + src
)
if fdroidserver.common.get_file_extension(src) == 'apk':
vercode, version = get_metadata_from_apk(app, build, src)
@ -783,16 +697,15 @@ def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball
def run_this_build(config, options, package_name, version_code):
"""run build for one specific version of an app localy
"""Run build for one specific version of an app localy.
:raises: various exceptions in case and of the pre-required conditions for the requested build are not met
"""
app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
# not sure if this makes any sense to change open file limits since we know
# that this script will only ever build one app
rlimit_check(1)
rlimit_check()
logging.info(
"Building version %s (%s) of %s"
@ -801,7 +714,7 @@ def run_this_build(config, options, package_name, version_code):
# init fdroid Popen wrapper
fdroidserver.common.set_FDroidPopen_env(build)
p, gradletasks, flavours_cmd = init_build_subprocess(app, build, config)
gradletasks = init_build(app, build, config)
sanitize_build_dir(app)
@ -817,14 +730,13 @@ def run_this_build(config, options, package_name, version_code):
execute_build_commands(app, build)
# Build native stuff if required...
execute_buildjni_commands(app, build)
ndk_path = install_ndk(build, config) # TODO: move to prepare step?
execute_buildjni_commands(app, build, ndk_path)
# Build the release...
p = execute_build___(app, build, config, gradletasks)
collect_build_output(app, build, p, flavours_cmd)
buildbuildbuild___(app, build, config, gradletasks)
p, bindir = execute_build(app, build, config, gradletasks)
check_build_success(app, build, p)
src = collect_output_again(app, build, p, flavours_cmd)
src = collect_build_output(app, build, p, bindir)
# Run a postbuild command if one is required...
execute_postbuild(app, build, src)
@ -835,6 +747,7 @@ def run_this_build(config, options, package_name, version_code):
move_build_output(app, build, src, tmp_dir)
def main():
parser = argparse.ArgumentParser(
description=_(
@ -844,7 +757,8 @@ def main():
)
)
parser.add_argument(
"APP_VERSION", help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)")
"APP_VERSION",
help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)"),
)
# fdroid args/opts boilerplate

2
fdroidserver/build_local_sudo.py
View File

@ -90,7 +90,6 @@ def main():
package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION)
app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
# intialize FDroidPopen
# TODO: remove once FDroidPopen is replaced with vm/container exec
fdroidserver.common.set_FDroidPopen_env(build)
@ -100,5 +99,6 @@ def main():
sudo_lock_root(app, build)
sudo_uninstall(app, build)
if __name__ == "__main__":
main()

55
fdroidserver/common.py
View File

@ -1148,7 +1148,24 @@ def get_head_commit_id(git_repo):
def setup_vcs(app):
"""Checkout code from VCS and return instance of vcs and the build dir."""
"""Create a VCS instance for given app.
This is a factory function that creates the correct type of VCS instance.
This doesn't checkout or clone any source code, it just creates a VCS
instance.
Parameters
----------
app
metadata app object
Returns
-------
vcs
VCS instance corresponding to passed app
build_dir
source code checkout directory for the supplied app
"""
build_dir = get_build_dir(app)
# Set up vcs interface and make sure we have the latest code...
@ -2213,7 +2230,41 @@ def getsrclib(spec, srclib_dir, basepath=False,
referencing it, which may be a subdirectory of the actual project. If
you want the base directory of the project, pass 'basepath=True'.
spec and srclib_dir are both strings, not pathlib.Path.
Parameters
----------
spec
srclib identifier (e.g. 'reproducible-apk-tools@v0.2.3').
must be string.
srclib_dir
base dir for holding checkouts of srclibs (usually './build/srclib').
must be a string.
basepath
changes the output of libdir to the base path, if set to True (default:
False)
raw
Don't sparese the spec instead use the unparsed spec as name, if set to
True (default: False)
prepare
Don't run `Prepare` commands in metadata, if set to False (default:
True)
preponly
Don't checkout the latest source code, if set to True (default: False)
refresh
Don't fetch latest source code from git remote, if set to False
(default: True)
build
metadata build object
Returns
-------
name
name of the srclib (e.g. 'mylib' when the spec is 'mylib@1.2.3')
number
number prefix from srclib spec (e.g. '7' when spec is '7:mylib@1.2.3')
(only used for ant builds)
libdir
(sub-)directory with the source code of this srclib (if basepath is set
this will ignore 'Subdir' from srclib metadata)
"""
number = None
subdir = None

2
fdroidserver/metadata.py
View File

@ -608,7 +608,7 @@ def read_metadata(appids={}, sort_by_time=False):
def read_build_metadata(package_name, version_code, check_disabled=True):
"""read 1 single metadata file from the file system + 1 singled out build
"""Read 1 single metadata file from the file system + 1 singled out build.
Parameters
----------