mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
Merge branch 'build-refactoring' into 'master'
draft: build overhault See merge request fdroid/fdroidserver!1486
This commit is contained in:
Michael Pöhn
commit
b09939d2b6
@ -56,6 +56,14 @@ COMMANDS = OrderedDict([
|
||||
])
|
||||
|
||||
|
||||
# commands that are not advertised as public api, intended for the use-case
|
||||
# of breaking down builds in buildbot into smaller steps
|
||||
COMMANDS_INTERNAL = [
|
||||
"build_local_prepare",
|
||||
"build_local_run",
|
||||
]
|
||||
|
||||
|
||||
def print_help(available_plugins=None):
|
||||
print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]"))
|
||||
print("")
|
||||
@ -136,7 +144,12 @@ def main():
|
||||
sys.exit(0)
|
||||
|
||||
command = sys.argv[1]
|
||||
if command not in COMMANDS and command not in available_plugins:
|
||||
command_not_found = (
|
||||
command not in COMMANDS
|
||||
and command not in COMMANDS_INTERNAL
|
||||
and command not in available_plugins
|
||||
)
|
||||
if command_not_found:
|
||||
if command in ('-h', '--help'):
|
||||
print_help(available_plugins=available_plugins)
|
||||
sys.exit(0)
|
||||
@ -186,7 +199,7 @@ def main():
|
||||
sys.argv[0] += ' ' + command
|
||||
|
||||
del sys.argv[1]
|
||||
if command in COMMANDS.keys():
|
||||
if command in COMMANDS.keys() or command in COMMANDS_INTERNAL:
|
||||
# import is named import_subcommand internally b/c import is reserved by Python
|
||||
command = 'import_subcommand' if command == 'import' else command
|
||||
mod = __import__('fdroidserver.' + command, None, None, [command])
|
||||
|
||||
84
fdroidserver/build_local_prepare.py
Normal file
84
fdroidserver/build_local_prepare.py
Normal file
@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# build.py - part of the FDroid server tools
|
||||
# Copyright (C) 2024, Michael Pöhn <michael@poehn.at>
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
# import logging
|
||||
import argparse
|
||||
|
||||
from fdroidserver import _
|
||||
import fdroidserver.common
|
||||
|
||||
|
||||
def log_tools_version(app, build, log_dir):
|
||||
log_path = os.path.join(
|
||||
log_dir, fdroidserver.common.get_toolsversion_logname(app, build)
|
||||
)
|
||||
with open(log_path, 'w') as f:
|
||||
f.write(fdroidserver.common.get_android_tools_version_log())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=_(
|
||||
"Download source code and initialize build environment "
|
||||
"for one specific build"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"APP_VERSION",
|
||||
help=_("app id and version code tuple 'APPID:VERCODE'"),
|
||||
)
|
||||
|
||||
# fdroid args/opts boilerplate
|
||||
fdroidserver.common.setup_global_opts(parser)
|
||||
options = fdroidserver.common.parse_args(parser)
|
||||
config = fdroidserver.common.get_config()
|
||||
|
||||
config # silence pyflakes
|
||||
|
||||
package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION)
|
||||
app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
|
||||
|
||||
# prepare folders for git/vcs checkout
|
||||
vcs, build_dir = fdroidserver.common.setup_vcs(app)
|
||||
srclib_dir = pathlib.Path('./build/srclib')
|
||||
extlib_dir = pathlib.Path('./build/extlib')
|
||||
log_dir = pathlib.Path('./logs')
|
||||
output_dir = pathlib.Path('./unsigned')
|
||||
for d in (srclib_dir, extlib_dir, log_dir, output_dir):
|
||||
d.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
# TODO: in the past this was only logged when running as 'fdroid build
|
||||
# --onserver', is this output still valuable or can we remove it?
|
||||
log_tools_version(app, build, log_dir)
|
||||
|
||||
# do git/vcs checkout
|
||||
fdroidserver.common.prepare_source(
|
||||
vcs, app, build, build_dir, str(srclib_dir), str(extlib_dir)
|
||||
)
|
||||
|
||||
# prepare for running cli commands
|
||||
# NOTE: unclear if this is required here
|
||||
# ndk_path = get_ndk_path(build)
|
||||
# fdroidserver.common.set_FDroidPopen_env(build)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
724
fdroidserver/build_local_run.py
Normal file
724
fdroidserver/build_local_run.py
Normal file
@ -0,0 +1,724 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import glob
|
||||
import shutil
|
||||
import logging
|
||||
import pathlib
|
||||
import tarfile
|
||||
import argparse
|
||||
import traceback
|
||||
|
||||
import fdroidserver.common
|
||||
import fdroidserver.metadata
|
||||
import fdroidserver.exception
|
||||
|
||||
from fdroidserver import _
|
||||
|
||||
|
||||
def rlimit_check(apps_count=1):
|
||||
"""Make sure linux is confgured to allow for enough simultaneously oepn files.
|
||||
|
||||
TODO: check if this is obsolete
|
||||
|
||||
Parameters
|
||||
----------
|
||||
apps_count
|
||||
In the past this used to be `len(apps)` In this context we're
|
||||
always buidling just one app so this is always 1
|
||||
"""
|
||||
try:
|
||||
import resource # not available on Windows
|
||||
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
if apps_count > soft:
|
||||
try:
|
||||
soft = apps_count * 2
|
||||
if soft > hard:
|
||||
soft = hard
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
|
||||
logging.debug(
|
||||
_('Set open file limit to {integer}').format(integer=soft)
|
||||
)
|
||||
except (OSError, ValueError) as e:
|
||||
logging.warning(_('Setting open file limit failed: ') + str(e))
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def install_ndk(build, config):
|
||||
"""Make sure the requested NDK version is or gets installed
|
||||
|
||||
TODO: check if this should be moved to a script that runs before starting
|
||||
the build. e.g. `build_local_prepare` or `build_local_sudo`
|
||||
|
||||
Parameters
|
||||
----------
|
||||
build
|
||||
Metadata build entry that's about the build and may contain the
|
||||
requested NDK version
|
||||
"""
|
||||
ndk_path = build.ndk_path()
|
||||
if build.ndk or (build.buildjni and build.buildjni != ['no']):
|
||||
if not ndk_path:
|
||||
logging.warning("Android NDK version '%s' could not be found!" % build.ndk)
|
||||
logging.warning("Configured versions:")
|
||||
for k, v in config['ndk_paths'].items():
|
||||
if k.endswith("_orig"):
|
||||
continue
|
||||
logging.warning(" %s: %s" % (k, v))
|
||||
fdroidserver.common.auto_install_ndk(build)
|
||||
elif not os.path.isdir(ndk_path):
|
||||
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
|
||||
raise fdroidserver.exception.FDroidException()
|
||||
return ndk_path
|
||||
|
||||
|
||||
def get_build_root_dir(app, build):
|
||||
if build.subdir:
|
||||
return os.path.join(fdroidserver.common.get_build_dir(app), build.subdir)
|
||||
return fdroidserver.common.get_build_dir(app)
|
||||
|
||||
|
||||
def transform_first_char(string, method):
|
||||
"""Use method() on the first character of string."""
|
||||
if len(string) == 0:
|
||||
return string
|
||||
if len(string) == 1:
|
||||
return method(string)
|
||||
return method(string[0]) + string[1:]
|
||||
|
||||
|
||||
def init_build(app, build, config):
|
||||
# We need to clean via the build tool in case the binary dirs are
|
||||
# different from the default ones
|
||||
root_dir = get_build_root_dir(app, build)
|
||||
|
||||
p = None
|
||||
gradletasks = []
|
||||
flavours_cmd = None
|
||||
|
||||
bmethod = build.build_method()
|
||||
if bmethod == 'maven':
|
||||
logging.info("Cleaning Maven project...")
|
||||
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
|
||||
|
||||
if '@' in build.maven:
|
||||
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
|
||||
maven_dir = os.path.normpath(maven_dir)
|
||||
else:
|
||||
maven_dir = root_dir
|
||||
|
||||
p = fdroidserver.common.FDroidPopen(cmd, cwd=maven_dir)
|
||||
|
||||
elif bmethod == 'gradle':
|
||||
|
||||
logging.info("Cleaning Gradle project...")
|
||||
|
||||
if build.preassemble:
|
||||
gradletasks += build.preassemble
|
||||
|
||||
flavours = build.gradle
|
||||
if flavours == ['yes']:
|
||||
flavours = []
|
||||
|
||||
flavours_cmd = ''.join(
|
||||
[transform_first_char(flav, str.upper) for flav in flavours]
|
||||
)
|
||||
|
||||
gradletasks += ['assemble' + flavours_cmd + 'Release']
|
||||
|
||||
cmd = [config['gradle']]
|
||||
if build.gradleprops:
|
||||
cmd += ['-P' + kv for kv in build.gradleprops]
|
||||
|
||||
cmd += ['clean']
|
||||
p = fdroidserver.common.FDroidPopen(
|
||||
cmd,
|
||||
cwd=root_dir,
|
||||
envs={
|
||||
"GRADLE_VERSION_DIR": config['gradle_version_dir'],
|
||||
"CACHEDIR": config['cachedir'],
|
||||
},
|
||||
)
|
||||
|
||||
elif bmethod == 'ant':
|
||||
logging.info("Cleaning Ant project...")
|
||||
p = fdroidserver.common.FDroidPopen(['ant', 'clean'], cwd=root_dir)
|
||||
|
||||
if p is not None and p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Error cleaning %s:%s" % (app.id, build.versionName), p.output
|
||||
)
|
||||
|
||||
return gradletasks, flavours_cmd
|
||||
|
||||
|
||||
def sanitize_build_dir(app):
|
||||
"""delete build output directories
|
||||
|
||||
This function deletes the default build/binary/target/... output
|
||||
directories for follwoing build tools: gradle, maven, ant, jni. It also
|
||||
deletes gradle-wrapper if present.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app
|
||||
The metadata of the app to sanitize
|
||||
"""
|
||||
build_dir = fdroidserver.common.get_build_dir(app)
|
||||
for root, dirs, files in os.walk(build_dir):
|
||||
|
||||
def del_dirs(dl):
|
||||
for d in dl:
|
||||
shutil.rmtree(os.path.join(root, d), ignore_errors=True)
|
||||
|
||||
def del_files(fl):
|
||||
for f in fl:
|
||||
if f in files:
|
||||
os.remove(os.path.join(root, f))
|
||||
|
||||
if any(
|
||||
f in files
|
||||
for f in [
|
||||
'build.gradle',
|
||||
'build.gradle.kts',
|
||||
'settings.gradle',
|
||||
'settings.gradle.kts',
|
||||
]
|
||||
):
|
||||
# Even when running clean, gradle stores task/artifact caches in
|
||||
# .gradle/ as binary files. To avoid overcomplicating the scanner,
|
||||
# manually delete them, just like `gradle clean` should have removed
|
||||
# the build/* dirs.
|
||||
del_dirs(
|
||||
[
|
||||
os.path.join('build', 'android-profile'),
|
||||
os.path.join('build', 'generated'),
|
||||
os.path.join('build', 'intermediates'),
|
||||
os.path.join('build', 'outputs'),
|
||||
os.path.join('build', 'reports'),
|
||||
os.path.join('build', 'tmp'),
|
||||
os.path.join('buildSrc', 'build'),
|
||||
'.gradle',
|
||||
]
|
||||
)
|
||||
del_files(['gradlew', 'gradlew.bat'])
|
||||
|
||||
if 'pom.xml' in files:
|
||||
del_dirs(['target'])
|
||||
|
||||
if any(
|
||||
f in files for f in ['ant.properties', 'project.properties', 'build.xml']
|
||||
):
|
||||
del_dirs(['bin', 'gen'])
|
||||
|
||||
if 'jni' in dirs:
|
||||
del_dirs(['obj'])
|
||||
|
||||
|
||||
def make_tarball(app, build, tmp_dir):
|
||||
build_dir = fdroidserver.common.get_build_dir(app)
|
||||
# Build the source tarball right before we build the release...
|
||||
logging.info("Creating source tarball...")
|
||||
tarname = fdroidserver.common.getsrcname(app, build)
|
||||
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
|
||||
|
||||
def tarexc(t):
|
||||
return (
|
||||
None
|
||||
if any(t.name.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
|
||||
else t
|
||||
)
|
||||
|
||||
tarball.add(build_dir, tarname, filter=tarexc)
|
||||
tarball.close()
|
||||
|
||||
|
||||
def execute_build_commands(app, build, srclibpaths):
|
||||
root_dir = get_build_root_dir(app, build)
|
||||
if build.build:
|
||||
logging.info("Running 'build' commands in %s" % root_dir)
|
||||
cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build)
|
||||
|
||||
# Substitute source library paths into commands...
|
||||
for name, number, libpath in srclibpaths:
|
||||
cmd = cmd.replace('$$' + name + '$$', os.path.join(os.getcwd(), libpath))
|
||||
|
||||
p = fdroidserver.common.FDroidPopen(
|
||||
['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir
|
||||
)
|
||||
|
||||
if p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Error running build command for %s:%s" % (app.id, build.versionName),
|
||||
p.output,
|
||||
)
|
||||
|
||||
|
||||
def checkout_srclibs(app, build):
|
||||
""""""
|
||||
build_dir = fdroidserver.common.get_build_dir(app)
|
||||
srclibpaths = []
|
||||
if build.srclibs:
|
||||
logging.info("Collecting source libraries")
|
||||
for lib in build.srclibs:
|
||||
srclibpaths.append(
|
||||
fdroidserver.common.getsrclib(
|
||||
lib, build_dir, prepare=False, refresh=False, build=build
|
||||
)
|
||||
)
|
||||
return srclibpaths
|
||||
|
||||
|
||||
def execute_buildjni_commands(app, build, ndk_path):
|
||||
root_dir = get_build_root_dir(app, build)
|
||||
if build.buildjni and build.buildjni != ['no']:
|
||||
logging.info("Building the native code")
|
||||
jni_components = build.buildjni
|
||||
|
||||
if jni_components == ['yes']:
|
||||
jni_components = ['']
|
||||
cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
|
||||
for d in jni_components:
|
||||
if d:
|
||||
logging.info("Building native code in '%s'" % d)
|
||||
else:
|
||||
logging.info("Building native code in the main project")
|
||||
manifest = os.path.join(root_dir, d, 'AndroidManifest.xml')
|
||||
if os.path.exists(manifest):
|
||||
# Read and write the whole AM.xml to fix newlines and avoid
|
||||
# the ndk r8c or later 'wordlist' errors. The outcome of this
|
||||
# under gnu/linux is the same as when using tools like
|
||||
# dos2unix, but the native python way is faster and will
|
||||
# work in non-unix systems.
|
||||
manifest_text = open(manifest, 'U').read()
|
||||
open(manifest, 'w').write(manifest_text)
|
||||
# In case the AM.xml read was big, free the memory
|
||||
del manifest_text
|
||||
p = fdroidserver.common.FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
|
||||
if p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"NDK build failed for %s:%s" % (app.id, build.versionName), p.output
|
||||
)
|
||||
|
||||
|
||||
def execute_build(app, build, config, gradletasks):
|
||||
build_dir = fdroidserver.common.get_build_dir(app)
|
||||
root_dir = get_build_root_dir(app, build)
|
||||
|
||||
p = None
|
||||
bindir = None
|
||||
bmethod = build.build_method()
|
||||
if bmethod == 'maven':
|
||||
logging.info("Building Maven project...")
|
||||
|
||||
if '@' in build.maven:
|
||||
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
|
||||
else:
|
||||
maven_dir = root_dir
|
||||
|
||||
mvncmd = [
|
||||
config['mvn3'],
|
||||
'-Dandroid.sdk.path=' + config['sdk_path'],
|
||||
'-Dmaven.jar.sign.skip=true',
|
||||
'-Dmaven.test.skip=true',
|
||||
'-Dandroid.sign.debug=false',
|
||||
'-Dandroid.release=true',
|
||||
'package',
|
||||
]
|
||||
if build.target:
|
||||
target = build.target.split('-')[1]
|
||||
fdroidserver.common.regsub_file(
|
||||
r'<platform>[0-9]*</platform>',
|
||||
r'<platform>%s</platform>' % target,
|
||||
os.path.join(root_dir, 'pom.xml'),
|
||||
)
|
||||
if '@' in build.maven:
|
||||
fdroidserver.common.regsub_file(
|
||||
r'<platform>[0-9]*</platform>',
|
||||
r'<platform>%s</platform>' % target,
|
||||
os.path.join(maven_dir, 'pom.xml'),
|
||||
)
|
||||
|
||||
p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir)
|
||||
|
||||
bindir = os.path.join(root_dir, 'target')
|
||||
|
||||
elif bmethod == 'gradle':
|
||||
logging.info("Building Gradle project...")
|
||||
|
||||
cmd = [config['gradle']]
|
||||
if build.gradleprops:
|
||||
cmd += ['-P' + kv for kv in build.gradleprops]
|
||||
|
||||
cmd += gradletasks
|
||||
|
||||
p = fdroidserver.common.FDroidPopen(
|
||||
cmd,
|
||||
cwd=root_dir,
|
||||
envs={
|
||||
"GRADLE_VERSION_DIR": config['gradle_version_dir'],
|
||||
"CACHEDIR": config['cachedir'],
|
||||
},
|
||||
)
|
||||
|
||||
elif bmethod == 'ant':
|
||||
logging.info("Building Ant project...")
|
||||
cmd = ['ant']
|
||||
if build.antcommands:
|
||||
cmd += build.antcommands
|
||||
else:
|
||||
cmd += ['release']
|
||||
p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir)
|
||||
|
||||
bindir = os.path.join(root_dir, 'bin')
|
||||
|
||||
return p, bindir
|
||||
|
||||
|
||||
def collect_build_output(app, build, p, flavours_cmd, bindir):
|
||||
root_dir = get_build_root_dir(app, build)
|
||||
|
||||
omethod = build.output_method()
|
||||
src = None
|
||||
if omethod == 'maven':
|
||||
stdout_apk = '\n'.join(
|
||||
[
|
||||
line
|
||||
for line in p.output.splitlines()
|
||||
if any(a in line for a in ('.apk', '.ap_', '.jar'))
|
||||
]
|
||||
)
|
||||
m = re.match(
|
||||
r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M
|
||||
)
|
||||
if not m:
|
||||
m = re.match(
|
||||
r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
|
||||
stdout_apk,
|
||||
re.S | re.M,
|
||||
)
|
||||
if not m:
|
||||
m = re.match(
|
||||
r'.*^\[INFO\] [^$]*aapt \[package,[^$]*'
|
||||
+ bindir
|
||||
+ r'/([^/]+)\.ap[_k][,\]]',
|
||||
stdout_apk,
|
||||
re.S | re.M,
|
||||
)
|
||||
|
||||
if not m:
|
||||
m = re.match(
|
||||
r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
|
||||
stdout_apk,
|
||||
re.S | re.M,
|
||||
)
|
||||
if not m:
|
||||
raise fdroidserver.exception.BuildException('Failed to find output')
|
||||
src = m.group(1)
|
||||
src = os.path.join(bindir, src) + '.apk'
|
||||
|
||||
elif omethod == 'gradle':
|
||||
src = None
|
||||
apk_dirs = [
|
||||
# gradle plugin >= 3.0
|
||||
os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'),
|
||||
# gradle plugin < 3.0 and >= 0.11
|
||||
os.path.join(root_dir, 'build', 'outputs', 'apk'),
|
||||
# really old path
|
||||
os.path.join(root_dir, 'build', 'apk'),
|
||||
]
|
||||
# If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
|
||||
# a subdirectory corresponding to the flavour command used, but with different
|
||||
# capitalization.
|
||||
if flavours_cmd:
|
||||
apk_dirs.append(
|
||||
os.path.join(
|
||||
root_dir,
|
||||
'build',
|
||||
'outputs',
|
||||
'apk',
|
||||
transform_first_char(flavours_cmd, str.lower),
|
||||
'release',
|
||||
)
|
||||
)
|
||||
for apks_dir in apk_dirs:
|
||||
for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
|
||||
apks = glob.glob(os.path.join(apks_dir, apkglob))
|
||||
|
||||
if len(apks) > 1:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
'More than one resulting apks found in %s' % apks_dir,
|
||||
'\n'.join(apks),
|
||||
)
|
||||
if len(apks) == 1:
|
||||
src = apks[0]
|
||||
break
|
||||
if src is not None:
|
||||
break
|
||||
|
||||
if src is None:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
'Failed to find any output apks'
|
||||
)
|
||||
|
||||
elif omethod == 'ant':
|
||||
stdout_apk = '\n'.join(
|
||||
[line for line in p.output.splitlines() if '.apk' in line]
|
||||
)
|
||||
src = re.match(
|
||||
r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M
|
||||
).group(1)
|
||||
src = os.path.join(bindir, src)
|
||||
elif omethod == 'raw':
|
||||
output_path = fdroidserver.common.replace_build_vars(build.output, build)
|
||||
globpath = os.path.join(root_dir, output_path)
|
||||
apks = glob.glob(globpath)
|
||||
if len(apks) > 1:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
'Multiple apks match %s' % globpath, '\n'.join(apks)
|
||||
)
|
||||
if len(apks) < 1:
|
||||
raise fdroidserver.exception.BuildException('No apks match %s' % globpath)
|
||||
src = os.path.normpath(apks[0])
|
||||
return src
|
||||
|
||||
|
||||
def check_build_success(app, build, p):
|
||||
build_dir = fdroidserver.common.get_build_dir(app)
|
||||
|
||||
if os.path.isdir(os.path.join(build_dir, '.git')):
|
||||
import git
|
||||
|
||||
commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
|
||||
else:
|
||||
commit_id = build.commit
|
||||
|
||||
if p is not None and p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
|
||||
p.output,
|
||||
)
|
||||
logging.info(
|
||||
"Successfully built version {versionName} of {appid} from {commit_id}".format(
|
||||
versionName=build.versionName, appid=app.id, commit_id=commit_id
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def execute_postbuild(app, build, src, srclibpaths):
|
||||
root_dir = get_build_root_dir(app, build)
|
||||
|
||||
if build.postbuild:
|
||||
logging.info(f"Running 'postbuild' commands in {root_dir}")
|
||||
cmd = fdroidserver.common.replace_config_vars("; ".join(build.postbuild), build)
|
||||
|
||||
# Substitute source library paths into commands...
|
||||
for name, number, libpath in srclibpaths:
|
||||
cmd = cmd.replace(f"$${name}$$", str(pathlib.Path.cwd() / libpath))
|
||||
|
||||
cmd = cmd.replace('$$OUT$$', str(pathlib.Path(src).resolve()))
|
||||
|
||||
p = fdroidserver.common.FDroidPopen(
|
||||
['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir
|
||||
)
|
||||
|
||||
if p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Error running postbuild command for " f"{app.id}:{build.versionName}",
|
||||
p.output,
|
||||
)
|
||||
|
||||
|
||||
def get_metadata_from_apk(app, build, apkfile):
|
||||
"""Get the required metadata from the built APK.
|
||||
|
||||
VersionName is allowed to be a blank string, i.e. ''
|
||||
|
||||
Parameters
|
||||
----------
|
||||
app
|
||||
The app metadata used to build the APK.
|
||||
build
|
||||
The build that resulted in the APK.
|
||||
apkfile
|
||||
The path of the APK file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
versionCode
|
||||
The versionCode from the APK or from the metadata is build.novcheck is
|
||||
set.
|
||||
versionName
|
||||
The versionName from the APK or from the metadata is build.novcheck is
|
||||
set.
|
||||
|
||||
Raises
|
||||
------
|
||||
:exc:`~fdroidserver.exception.BuildException`
|
||||
If native code should have been built but was not packaged, no version
|
||||
information or no package ID could be found or there is a mismatch
|
||||
between the package ID in the metadata and the one found in the APK.
|
||||
"""
|
||||
appid, versionCode, versionName = fdroidserver.common.get_apk_id(apkfile)
|
||||
native_code = fdroidserver.common.get_native_code(apkfile)
|
||||
|
||||
if build.buildjni and build.buildjni != ['no'] and not native_code:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Native code should have been built but none was packaged"
|
||||
)
|
||||
if build.novcheck:
|
||||
versionCode = build.versionCode
|
||||
versionName = build.versionName
|
||||
if not versionCode or versionName is None:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Could not find version information in build in output"
|
||||
)
|
||||
if not appid:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Could not find package ID in output"
|
||||
)
|
||||
if appid != app.id:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Wrong package ID - build " + appid + " but expected " + app.id
|
||||
)
|
||||
|
||||
return versionCode, versionName
|
||||
|
||||
|
||||
def validate_build_artifacts(app, build, src):
|
||||
# Make sure it's not debuggable...
|
||||
if fdroidserver.common.is_debuggable_or_testOnly(src):
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"%s: debuggable or testOnly set in AndroidManifest.xml" % src
|
||||
)
|
||||
|
||||
# By way of a sanity check, make sure the version and version
|
||||
# code in our new APK match what we expect...
|
||||
logging.debug("Checking " + src)
|
||||
if not os.path.exists(src):
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Unsigned APK is not at expected location of " + src
|
||||
)
|
||||
|
||||
if fdroidserver.common.get_file_extension(src) == 'apk':
|
||||
vercode, version = get_metadata_from_apk(app, build, src)
|
||||
if version != build.versionName or vercode != build.versionCode:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
(
|
||||
"Unexpected version/version code in output;"
|
||||
" APK: '%s' / '%d', "
|
||||
" Expected: '%s' / '%d'"
|
||||
)
|
||||
% (version, vercode, build.versionName, build.versionCode)
|
||||
)
|
||||
|
||||
|
||||
def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball=False):
|
||||
tarname = fdroidserver.common.getsrcname(app, build)
|
||||
|
||||
# Copy the unsigned APK to our destination directory for further
|
||||
# processing (by publish.py)...
|
||||
dest = os.path.join(
|
||||
output_dir,
|
||||
fdroidserver.common.get_release_filename(
|
||||
app, build, fdroidserver.common.get_file_extension(src)
|
||||
),
|
||||
)
|
||||
shutil.copyfile(src, dest)
|
||||
|
||||
# Move the source tarball into the output directory...
|
||||
if output_dir != tmp_dir and not notarball:
|
||||
shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname))
|
||||
|
||||
|
||||
def run_this_build(config, options, package_name, version_code):
|
||||
"""run build for one specific version of an app localy
|
||||
|
||||
:raises: various exceptions in case and of the pre-required conditions for the requested build are not met
|
||||
"""
|
||||
|
||||
app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
|
||||
|
||||
# not sure if this makes any sense to change open file limits since we know
|
||||
# that this script will only ever build one app
|
||||
rlimit_check()
|
||||
|
||||
logging.info(
|
||||
"Building version %s (%s) of %s"
|
||||
% (build.versionName, build.versionCode, app.id)
|
||||
)
|
||||
|
||||
# init fdroid Popen wrapper
|
||||
fdroidserver.common.set_FDroidPopen_env(build)
|
||||
gradletasks, flavours_cmd = init_build(app, build, config)
|
||||
|
||||
sanitize_build_dir(app)
|
||||
|
||||
# this is where we'd call scanner.scan_source() in old build.py
|
||||
|
||||
# create tarball before building
|
||||
# consider this optional?
|
||||
tmp_dir = pathlib.Path("./tmp")
|
||||
tmp_dir.mkdir(exist_ok=True)
|
||||
make_tarball(app, build, tmp_dir)
|
||||
|
||||
# Run a build command if one is required...
|
||||
srclibpaths = checkout_srclibs(app, build)
|
||||
execute_build_commands(app, build, srclibpaths)
|
||||
|
||||
# Build native stuff if required...
|
||||
ndk_path = install_ndk(build, config) # TODO: move to prepare step?
|
||||
execute_buildjni_commands(app, build, ndk_path)
|
||||
|
||||
# Build the release...
|
||||
p, bindir = execute_build(app, build, config, gradletasks)
|
||||
check_build_success(app, build, p)
|
||||
src = collect_build_output(app, build, p, flavours_cmd, bindir)
|
||||
|
||||
# Run a postbuild command if one is required...
|
||||
execute_postbuild(app, build, src, srclibpaths)
|
||||
|
||||
validate_build_artifacts(app, build, src)
|
||||
|
||||
# this is where we'd call scanner.scan_binary() in old build.py
|
||||
|
||||
move_build_output(app, build, src, tmp_dir)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=_(
|
||||
"Build one specific in app. This command "
|
||||
"assumes all required build tools are installed and "
|
||||
"configured."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"APP_VERSION",
|
||||
help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)"),
|
||||
)
|
||||
|
||||
# fdroid args/opts boilerplate
|
||||
fdroidserver.common.setup_global_opts(parser)
|
||||
options = fdroidserver.common.parse_args(parser)
|
||||
config = fdroidserver.common.get_config()
|
||||
|
||||
try:
|
||||
# read build target package name and version code from CLI arguments
|
||||
package_name, version_code = fdroidserver.common.split_pkg_arg(
|
||||
options.APP_VERSION
|
||||
)
|
||||
# trigger the build
|
||||
run_this_build(config, options, package_name, version_code)
|
||||
except Exception as e:
|
||||
if options.verbose:
|
||||
traceback.print_exc()
|
||||
else:
|
||||
print(e)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
104
fdroidserver/build_local_sudo.py
Normal file
104
fdroidserver/build_local_sudo.py
Normal file
@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# build.py - part of the FDroid server tools
|
||||
# Copyright (C) 2024, Michael Pöhn <michael@poehn.at>
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
from fdroidserver import _
|
||||
import fdroidserver.common
|
||||
|
||||
|
||||
def sudo_run(app, build):
|
||||
# before doing anything, run the sudo commands to setup the VM
|
||||
if build.sudo:
|
||||
logging.info("Running 'sudo' commands in %s" % os.getcwd())
|
||||
|
||||
p = fdroidserver.common.FDroidPopen(
|
||||
[
|
||||
'sudo',
|
||||
'DEBIAN_FRONTEND=noninteractive',
|
||||
'bash',
|
||||
'-e',
|
||||
'-u',
|
||||
'-o',
|
||||
'pipefail',
|
||||
'-x',
|
||||
'-c',
|
||||
'; '.join(build.sudo),
|
||||
]
|
||||
)
|
||||
if p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Error running sudo command for %s:%s" % (app.id, build.versionName),
|
||||
p.output,
|
||||
)
|
||||
|
||||
|
||||
def sudo_lock_root(app, build):
|
||||
p = fdroidserver.common.FDroidPopen(['sudo', 'passwd', '--lock', 'root'])
|
||||
if p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Error locking root account for %s:%s" % (app.id, build.versionName),
|
||||
p.output,
|
||||
)
|
||||
|
||||
|
||||
def sudo_uninstall(app, build):
|
||||
p = fdroidserver.common.FDroidPopen(
|
||||
['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo']
|
||||
)
|
||||
if p.returncode != 0:
|
||||
raise fdroidserver.exception.BuildException(
|
||||
"Error removing sudo for %s:%s" % (app.id, build.versionName), p.output
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=_(
|
||||
"""Run sudo commands """
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"APP_VERSION",
|
||||
help=_("app id and version code tuple 'APPID:VERCODE'"),
|
||||
)
|
||||
|
||||
# fdroid args/opts boilerplate
|
||||
fdroidserver.common.setup_global_opts(parser)
|
||||
options = fdroidserver.common.parse_args(parser)
|
||||
config = fdroidserver.common.get_config()
|
||||
config # silcense pyflakes
|
||||
|
||||
package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION)
|
||||
app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
|
||||
|
||||
|
||||
# intialize FDroidPopen
|
||||
# TODO: remove once FDroidPopen is replaced with vm/container exec
|
||||
fdroidserver.common.set_FDroidPopen_env(build)
|
||||
|
||||
# run sudo stuff
|
||||
sudo_run(app, build)
|
||||
sudo_lock_root(app, build)
|
||||
sudo_uninstall(app, build)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -870,6 +870,27 @@ def get_local_metadata_files():
|
||||
return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
|
||||
|
||||
|
||||
def split_pkg_arg(appid_versionCode_pair):
|
||||
"""Split 'appid:versionCode' pair into 2 separate values safely.
|
||||
|
||||
:raises ValueError: if argument is not parseable
|
||||
:return: (appid, versionCode) tuple with the 2 parsed values
|
||||
"""
|
||||
tokens = appid_versionCode_pair.split(":")
|
||||
if len(tokens) != 2:
|
||||
raise ValueError(
|
||||
_("'{}' is not a valid pair of the form appId:versionCode pair").format(
|
||||
appid_versionCode_pair
|
||||
)
|
||||
)
|
||||
if not is_valid_package_name(tokens[0]):
|
||||
raise ValueError(
|
||||
_("'{}' does not start with a valid appId").format(appid_versionCode_pair)
|
||||
)
|
||||
versionCode = version_code_string_to_int(tokens[1])
|
||||
return tokens[0], versionCode
|
||||
|
||||
|
||||
def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
|
||||
"""No summary.
|
||||
|
||||
|
||||
@ -607,6 +607,45 @@ def read_metadata(appids={}, sort_by_time=False):
|
||||
return apps
|
||||
|
||||
|
||||
def read_build_metadata(package_name, version_code, check_disabled=True):
|
||||
"""read 1 single metadata file from the file system + 1 singled out build
|
||||
|
||||
Parameters
|
||||
----------
|
||||
package_name
|
||||
appid of the metadata supposed to be loaded (e.g. 'org.fdroid.fdroid')
|
||||
version_code
|
||||
android integer version identifier of a build (e.g. 1234)
|
||||
check_disabled
|
||||
If True this function will raise an exception in case the disabled flag
|
||||
on the build or the metadata file is set.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Tuple
|
||||
A tuple of (metadata, build) dictionsaries, containing the pared data from
|
||||
the metadata file.
|
||||
|
||||
Raises
|
||||
------
|
||||
MetaDataException
|
||||
If parsing or selecting the requested version fails.
|
||||
"""
|
||||
m = read_metadata({package_name: version_code})
|
||||
if len(m) != 1 or package_name not in m:
|
||||
raise MetaDataException(f"Could not read metadata for '{package_name}:{version_code}' (metadata file might not be present or correct)")
|
||||
if check_disabled and m[package_name].get("Disabled"):
|
||||
raise MetaDataException(f"'{package_name}' disbaled in metadata file.")
|
||||
if "Builds" not in m[package_name]:
|
||||
raise MetaDataException(f"Cound not find 'Builds' section in '{package_name}' metadata file")
|
||||
for b in m[package_name]["Builds"]:
|
||||
if b['versionCode'] == version_code:
|
||||
if check_disabled and b.get('disable'):
|
||||
raise MetaDataException(f"'{package_name}:{version_code}' disabled in metadata file")
|
||||
return m[package_name], b
|
||||
raise MetaDataException(f"Could not find '{version_code}' in 'Builds' section of 'package_name' metadata file")
|
||||
|
||||
|
||||
def parse_metadata(metadatapath):
|
||||
"""Parse metadata file, also checking the source repo for .fdroid.yml.
|
||||
|
||||
|
||||
69
tests/build_local_run.TestCase
Executable file
69
tests/build_local_run.TestCase
Executable file
@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
# import json
|
||||
# import shutil
|
||||
# import logging
|
||||
import inspect
|
||||
# import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
localmodule = os.path.realpath(
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
||||
)
|
||||
print('localmodule: ' + localmodule)
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, localmodule)
|
||||
|
||||
import fdroidserver.build_local_run
|
||||
import fdroidserver.metadata
|
||||
|
||||
|
||||
class GetBuildRootDirTest(unittest.TestCase):
|
||||
def test_no_subdir(self):
|
||||
app = unittest.mock.MagicMock()
|
||||
app.id = "fake.app"
|
||||
build = unittest.mock.MagicMock()
|
||||
build.subdir = None
|
||||
|
||||
gbdmock = unittest.mock.Mock(return_value="ok")
|
||||
with unittest.mock.patch("fdroidserver.common.get_build_dir", gbdmock):
|
||||
r = fdroidserver.build_local_run.get_build_root_dir(app, build)
|
||||
|
||||
gbdmock.assert_called_once_with(app)
|
||||
self.assertEqual(r, "ok")
|
||||
|
||||
def test_subdir(self):
|
||||
app = unittest.mock.MagicMock()
|
||||
app.id = "fake.app"
|
||||
build = unittest.mock.MagicMock()
|
||||
build.subdir = "abc/def"
|
||||
|
||||
gbdmock = unittest.mock.Mock(return_value="ok")
|
||||
with unittest.mock.patch("fdroidserver.common.get_build_dir", gbdmock):
|
||||
r = fdroidserver.build_local_run.get_build_root_dir(app, build)
|
||||
|
||||
gbdmock.assert_called_once_with(app)
|
||||
self.assertEqual(r, "ok/abc/def")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Spew out even more information than normal",
|
||||
)
|
||||
fdroidserver.common.options = fdroidserver.common.parse_args(parser)
|
||||
|
||||
newSuite = unittest.TestSuite()
|
||||
newSuite.addTest(unittest.makeSuite(GetBuildRootDirTest))
|
||||
unittest.main(failfast=False)
|
||||
Loading…
Reference in New Issue
Block a user