Merge branch 'build-refactoring' into 'master'

draft: build overhault

See merge request fdroid/fdroidserver!1486

fdroidserver/__main__.py

@@ -56,6 +56,15 @@ COMMANDS = OrderedDict([
 ])
 
 
+# commands that are not advertised as public api, intended for the use-case
+# of breaking down builds in buildbot into smaller steps
+COMMANDS_INTERNAL = [
+    "build_local_prepare",
+    "build_local_run",
+    "build_local_sudo",
+]
+
+
 def print_help(available_plugins=None):
     print(_("usage: ") + _("fdroid [<command>] [-h|--help|--version|<args>]"))
     print("")
@@ -136,7 +145,12 @@ def main():
         sys.exit(0)
 
     command = sys.argv[1]
-    if command not in COMMANDS and command not in available_plugins:
+    command_not_found = (
+        command not in COMMANDS
+        and command not in COMMANDS_INTERNAL
+        and command not in available_plugins
+    )
+    if command_not_found:
         if command in ('-h', '--help'):
             print_help(available_plugins=available_plugins)
             sys.exit(0)
@@ -186,7 +200,7 @@ def main():
     sys.argv[0] += ' ' + command
 
     del sys.argv[1]
-    if command in COMMANDS.keys():
+    if command in COMMANDS.keys() or command in COMMANDS_INTERNAL:
         # import is named import_subcommand internally b/c import is reserved by Python
         command = 'import_subcommand' if command == 'import' else command
         mod = __import__('fdroidserver.' + command, None, None, [command])

fdroidserver/build_local_prepare.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+#
+# build.py - part of the FDroid server tools
+# Copyright (C) 2024, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import pathlib
+# import logging
+import argparse
+
+from fdroidserver import _
+import fdroidserver.common
+
+
+def log_tools_version(app, build, log_dir):
+    log_path = os.path.join(
+        log_dir, fdroidserver.common.get_toolsversion_logname(app, build)
+    )
+    with open(log_path, 'w') as f:
+        f.write(fdroidserver.common.get_android_tools_version_log())
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=_(
+            "Download source code and initialize build environment "
+            "for one specific build"
+        ),
+    )
+    parser.add_argument(
+        "APP_VERSION",
+        help=_("app id and version code tuple 'APPID:VERCODE'"),
+    )
+
+    # fdroid args/opts boilerplate
+    fdroidserver.common.setup_global_opts(parser)
+    options = fdroidserver.common.parse_args(parser)
+    config = fdroidserver.common.get_config()
+
+    config  # silence pyflakes
+
+    package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION)
+    app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
+
+    # prepare folders for git/vcs checkout
+    vcs, build_dir = fdroidserver.common.setup_vcs(app)
+    srclib_dir = pathlib.Path('./build/srclib')
+    extlib_dir = pathlib.Path('./build/extlib')
+    log_dir = pathlib.Path('./logs')
+    output_dir = pathlib.Path('./unsigned')
+    for d in (srclib_dir, extlib_dir, log_dir, output_dir):
+        d.mkdir(exist_ok=True, parents=True)
+
+    # TODO: in the past this was only logged when running as 'fdroid build
+    # --onserver', is this output still valuable or can we remove it?
+    log_tools_version(app, build, log_dir)
+
+    # do git/vcs checkout
+    fdroidserver.common.prepare_source(
+        vcs, app, build, build_dir, str(srclib_dir), str(extlib_dir)
+    )
+
+    # prepare for running cli commands
+    # NOTE: unclear if this is required here
+    # ndk_path = get_ndk_path(build)
+    # fdroidserver.common.set_FDroidPopen_env(build)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/build_local_run.py

@@ -0,0 +1,785 @@
+import os
+import re
+import sys
+import glob
+import shutil
+import logging
+import pathlib
+import tarfile
+import argparse
+import traceback
+
+import fdroidserver.common
+import fdroidserver.metadata
+import fdroidserver.exception
+
+from fdroidserver import _
+
+
+def rlimit_check(apps_count=1):
+    """Make sure linux is confgured to allow for enough simultaneously oepn files.
+
+    TODO: check if this is obsolete
+
+    Parameters
+    ----------
+    apps_count
+        In the past this used to be `len(apps)` In this context we're
+        always buidling just one app so this is always 1
+    """
+    try:
+        import resource  # not available on Windows
+
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if apps_count > soft:
+            try:
+                soft = apps_count * 2
+                if soft > hard:
+                    soft = hard
+                resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+                logging.debug(
+                    _('Set open file limit to {integer}').format(integer=soft)
+                )
+            except (OSError, ValueError) as e:
+                logging.warning(_('Setting open file limit failed: ') + str(e))
+    except ImportError:
+        pass
+
+
+def install_ndk(build, config):
+    """Make sure the requested NDK version is or gets installed.
+
+    TODO: check if this should be moved to a script that runs before starting
+    the build. e.g. `build_local_prepare` or `build_local_sudo`
+
+    Parameters
+    ----------
+    build
+        Metadata build entry that's about the build and may contain the
+        requested NDK version
+    config
+        dictonariy holding config/default data from `./config.yml`
+    """
+    ndk_path = build.ndk_path()
+    if build.ndk or (build.buildjni and build.buildjni != ['no']):
+        if not ndk_path:
+            logging.warning("Android NDK version '%s' could not be found!" % build.ndk)
+            logging.warning("Configured versions:")
+            for k, v in config['ndk_paths'].items():
+                if k.endswith("_orig"):
+                    continue
+                logging.warning("  %s: %s" % (k, v))
+            fdroidserver.common.auto_install_ndk(build)
+        elif not os.path.isdir(ndk_path):
+            logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
+            raise fdroidserver.exception.FDroidException()
+    return ndk_path
+
+
+def get_build_root_dir(app, build):
+    if build.subdir:
+        return os.path.join(fdroidserver.common.get_build_dir(app), build.subdir)
+    return fdroidserver.common.get_build_dir(app)
+
+
+def transform_first_char(string, method):
+    """Use method() on the first character of string."""
+    if len(string) == 0:
+        return string
+    if len(string) == 1:
+        return method(string)
+    return method(string[0]) + string[1:]
+
+
+def get_flavours_cmd(build):
+    """Get flavor string, preformatted for gradle cli.
+
+    Reads build flavors form metadata if any and reformats and concatenates
+    them to be ready for use as CLI arguments to gradle. This will treat the
+    vlue 'yes' as if there were not particular build flavor selected.
+
+    Parameters
+    ----------
+    build
+        The metadata build entry you'd like to read flavors from
+
+    Returns
+    -------
+    A string containing the build flavor for this build. If it's the default
+    flavor ("yes" in metadata) this returns an empty string. Returns None if
+    it's not a gradle build.
+    """
+    flavours = build.gradle
+
+    if flavours == ['yes']:
+        flavours = []
+
+    flavours_cmd = ''.join(
+        [transform_first_char(flav, str.upper) for flav in flavours]
+    )
+
+    return flavours_cmd
+
+
+def init_build(app, build, config):
+    root_dir = get_build_root_dir(app, build)
+
+    p = None
+    gradletasks = []
+
+    # We need to clean via the build tool in case the binary dirs are
+    # different from the default ones
+
+    bmethod = build.build_method()
+    if bmethod == 'maven':
+        logging.info("Cleaning Maven project...")
+        cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
+
+        if '@' in build.maven:
+            maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
+            maven_dir = os.path.normpath(maven_dir)
+        else:
+            maven_dir = root_dir
+
+        p = fdroidserver.common.FDroidPopen(cmd, cwd=maven_dir)
+
+    elif bmethod == 'gradle':
+
+        logging.info("Cleaning Gradle project...")
+
+        if build.preassemble:
+            gradletasks += build.preassemble
+
+        flavours_cmd = get_flavours_cmd(build)
+
+        gradletasks += ['assemble' + flavours_cmd + 'Release']
+
+        cmd = [config['gradle']]
+        if build.gradleprops:
+            cmd += ['-P' + kv for kv in build.gradleprops]
+
+        cmd += ['clean']
+        p = fdroidserver.common.FDroidPopen(
+            cmd,
+            cwd=root_dir,
+            envs={
+                "GRADLE_VERSION_DIR": config['gradle_version_dir'],
+                "CACHEDIR": config['cachedir'],
+            },
+        )
+
+    elif bmethod == 'ant':
+        logging.info("Cleaning Ant project...")
+        p = fdroidserver.common.FDroidPopen(['ant', 'clean'], cwd=root_dir)
+
+    if p is not None and p.returncode != 0:
+        raise fdroidserver.exception.BuildException(
+            "Error cleaning %s:%s" % (app.id, build.versionName), p.output
+        )
+
+    return gradletasks
+
+
+def sanitize_build_dir(app):
+    """Delete build output directories.
+
+    This function deletes the default build/binary/target/... output
+    directories for follwoing build tools: gradle, maven, ant, jni. It also
+    deletes gradle-wrapper if present. It just uses parths, hardcoded here,
+    it doesn't call and build system clean routines.
+
+    Parameters
+    ----------
+    app
+        The metadata of the app to sanitize
+    """
+    build_dir = fdroidserver.common.get_build_dir(app)
+    for root, dirs, files in os.walk(build_dir):
+
+        def del_dirs(dl):
+            for d in dl:
+                shutil.rmtree(os.path.join(root, d), ignore_errors=True)
+
+        def del_files(fl):
+            for f in fl:
+                if f in files:
+                    os.remove(os.path.join(root, f))
+
+        if any(
+            f in files
+            for f in [
+                'build.gradle',
+                'build.gradle.kts',
+                'settings.gradle',
+                'settings.gradle.kts',
+            ]
+        ):
+            # Even when running clean, gradle stores task/artifact caches in
+            # .gradle/ as binary files. To avoid overcomplicating the scanner,
+            # manually delete them, just like `gradle clean` should have removed
+            # the build/* dirs.
+            del_dirs(
+                [
+                    os.path.join('build', 'android-profile'),
+                    os.path.join('build', 'generated'),
+                    os.path.join('build', 'intermediates'),
+                    os.path.join('build', 'outputs'),
+                    os.path.join('build', 'reports'),
+                    os.path.join('build', 'tmp'),
+                    os.path.join('buildSrc', 'build'),
+                    '.gradle',
+                ]
+            )
+            del_files(['gradlew', 'gradlew.bat'])
+
+        if 'pom.xml' in files:
+            del_dirs(['target'])
+
+        if any(
+            f in files for f in ['ant.properties', 'project.properties', 'build.xml']
+        ):
+            del_dirs(['bin', 'gen'])
+
+        if 'jni' in dirs:
+            del_dirs(['obj'])
+
+
+def make_tarball(app, build, tmp_dir):
+    build_dir = fdroidserver.common.get_build_dir(app)
+    # Build the source tarball right before we build the release...
+    logging.info("Creating source tarball...")
+    tarname = fdroidserver.common.getsrcname(app, build)
+    tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
+
+    def tarexc(t):
+        return (
+            None
+            if any(t.name.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
+            else t
+        )
+
+    tarball.add(build_dir, tarname, filter=tarexc)
+    tarball.close()
+
+
+def execute_build_commands(app, build):
+    """Execute `bulid` commands if present in metadata.
+
+    see: https://f-droid.org/docs/Build_Metadata_Reference/#build_build
+
+    Parameters
+    ----------
+    app
+        metadata app object
+    build
+        metadata build object
+    """
+    root_dir = get_build_root_dir(app, build)
+    srclibpaths = get_srclibpaths(app, build)
+
+    if build.build:
+        logging.info("Running 'build' commands in %s" % root_dir)
+        cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build)
+
+        # Substitute source library paths into commands...
+        for name, number, libpath in srclibpaths:
+            cmd = cmd.replace('$$' + name + '$$', os.path.join(os.getcwd(), libpath))
+
+        p = fdroidserver.common.FDroidPopen(
+            ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir
+        )
+
+        if p.returncode != 0:
+            raise fdroidserver.exception.BuildException(
+                "Error running build command for %s:%s" % (app.id, build.versionName),
+                p.output,
+            )
+
+
+def get_srclibpaths(app, build):
+    """Get srclibpaths list of tuples.
+
+    This will just assemble the srclibpaths list of tuples, it won't fetch
+    or checkout any source code, identical to return value of
+    common.prepare_souce().
+
+    Parameters
+    ----------
+    app
+        metadata app object
+    build
+        metadata build object
+
+    Returns
+    -------
+    List of srclibpath tuples
+    """
+    vcs, _ = fdroidserver.common.setup_vcs(app)
+
+    srclibpaths = []
+    if build.srclibs:
+        logging.info("Collecting source libraries")
+        for lib in build.srclibs:
+            srclibpaths.append(
+                fdroidserver.common.getsrclib(
+                    lib, "./build/srclib", prepare=False, refresh=False, build=build
+                )
+            )
+
+    basesrclib = vcs.getsrclib()
+    if basesrclib:
+        srclibpaths.append(basesrclib)
+
+    return srclibpaths
+
+
+def execute_buildjni_commands(app, build, ndk_path):
+    root_dir = get_build_root_dir(app, build)
+    if build.buildjni and build.buildjni != ['no']:
+        logging.info("Building the native code")
+        jni_components = build.buildjni
+
+        if jni_components == ['yes']:
+            jni_components = ['']
+        cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
+        for d in jni_components:
+            if d:
+                logging.info("Building native code in '%s'" % d)
+            else:
+                logging.info("Building native code in the main project")
+            manifest = os.path.join(root_dir, d, 'AndroidManifest.xml')
+            if os.path.exists(manifest):
+                # Read and write the whole AM.xml to fix newlines and avoid
+                # the ndk r8c or later 'wordlist' errors. The outcome of this
+                # under gnu/linux is the same as when using tools like
+                # dos2unix, but the native python way is faster and will
+                # work in non-unix systems.
+                manifest_text = open(manifest, 'U').read()
+                open(manifest, 'w').write(manifest_text)
+                # In case the AM.xml read was big, free the memory
+                del manifest_text
+            p = fdroidserver.common.FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
+            if p.returncode != 0:
+                raise fdroidserver.exception.BuildException(
+                    "NDK build failed for %s:%s" % (app.id, build.versionName), p.output
+                )
+
+
+def execute_build(app, build, config, gradletasks):
+    root_dir = get_build_root_dir(app, build)
+
+    p = None
+    bindir = None
+    bmethod = build.build_method()
+    if bmethod == 'maven':
+        logging.info("Building Maven project...")
+
+        if '@' in build.maven:
+            maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
+        else:
+            maven_dir = root_dir
+
+        mvncmd = [
+            config['mvn3'],
+            '-Dandroid.sdk.path=' + config['sdk_path'],
+            '-Dmaven.jar.sign.skip=true',
+            '-Dmaven.test.skip=true',
+            '-Dandroid.sign.debug=false',
+            '-Dandroid.release=true',
+            'package',
+        ]
+        if build.target:
+            target = build.target.split('-')[1]
+            fdroidserver.common.regsub_file(
+                r'<platform>[0-9]*</platform>',
+                r'<platform>%s</platform>' % target,
+                os.path.join(root_dir, 'pom.xml'),
+            )
+            if '@' in build.maven:
+                fdroidserver.common.regsub_file(
+                    r'<platform>[0-9]*</platform>',
+                    r'<platform>%s</platform>' % target,
+                    os.path.join(maven_dir, 'pom.xml'),
+                )
+
+        p = fdroidserver.common.FDroidPopen(mvncmd, cwd=maven_dir)
+
+        bindir = os.path.join(root_dir, 'target')
+
+    elif bmethod == 'gradle':
+        logging.info("Building Gradle project...")
+
+        cmd = [config['gradle']]
+        if build.gradleprops:
+            cmd += ['-P' + kv for kv in build.gradleprops]
+
+        cmd += gradletasks
+
+        p = fdroidserver.common.FDroidPopen(
+            cmd,
+            cwd=root_dir,
+            envs={
+                "GRADLE_VERSION_DIR": config['gradle_version_dir'],
+                "CACHEDIR": config['cachedir'],
+            },
+        )
+
+    elif bmethod == 'ant':
+        logging.info("Building Ant project...")
+        cmd = ['ant']
+        if build.antcommands:
+            cmd += build.antcommands
+        else:
+            cmd += ['release']
+        p = fdroidserver.common.FDroidPopen(cmd, cwd=root_dir)
+
+        bindir = os.path.join(root_dir, 'bin')
+
+    return p, bindir
+
+
+def collect_build_output(app, build, p, bindir):
+    root_dir = get_build_root_dir(app, build)
+
+    omethod = build.output_method()
+    src = None
+    if omethod == 'maven':
+        stdout_apk = '\n'.join(
+            [
+                line
+                for line in p.output.splitlines()
+                if any(a in line for a in ('.apk', '.ap_', '.jar'))
+            ]
+        )
+        m = re.match(
+            r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M
+        )
+        if not m:
+            m = re.match(
+                r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
+                stdout_apk,
+                re.S | re.M,
+            )
+        if not m:
+            m = re.match(
+                r'.*^\[INFO\] [^$]*aapt \[package,[^$]*'
+                + bindir
+                + r'/([^/]+)\.ap[_k][,\]]',
+                stdout_apk,
+                re.S | re.M,
+            )
+
+        if not m:
+            m = re.match(
+                r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
+                stdout_apk,
+                re.S | re.M,
+            )
+        if not m:
+            raise fdroidserver.exception.BuildException('Failed to find output')
+        src = m.group(1)
+        src = os.path.join(bindir, src) + '.apk'
+
+    elif omethod == 'gradle':
+        src = None
+        apk_dirs = [
+            # gradle plugin >= 3.0
+            os.path.join(root_dir, 'build', 'outputs', 'apk', 'release'),
+            # gradle plugin < 3.0 and >= 0.11
+            os.path.join(root_dir, 'build', 'outputs', 'apk'),
+            # really old path
+            os.path.join(root_dir, 'build', 'apk'),
+        ]
+        # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
+        # a subdirectory corresponding to the flavour command used, but with different
+        # capitalization.
+        flavours_cmd = get_flavours_cmd(build)
+        if flavours_cmd:
+            apk_dirs.append(
+                os.path.join(
+                    root_dir,
+                    'build',
+                    'outputs',
+                    'apk',
+                    transform_first_char(flavours_cmd, str.lower),
+                    'release',
+                )
+            )
+        for apks_dir in apk_dirs:
+            for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']:
+                apks = glob.glob(os.path.join(apks_dir, apkglob))
+
+                if len(apks) > 1:
+                    raise fdroidserver.exception.BuildException(
+                        'More than one resulting apks found in %s' % apks_dir,
+                        '\n'.join(apks),
+                    )
+                if len(apks) == 1:
+                    src = apks[0]
+                    break
+            if src is not None:
+                break
+
+        if src is None:
+            raise fdroidserver.exception.BuildException(
+                'Failed to find any output apks'
+            )
+
+    elif omethod == 'ant':
+        stdout_apk = '\n'.join(
+            [line for line in p.output.splitlines() if '.apk' in line]
+        )
+        src = re.match(
+            r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M
+        ).group(1)
+        src = os.path.join(bindir, src)
+    elif omethod == 'raw':
+        output_path = fdroidserver.common.replace_build_vars(build.output, build)
+        globpath = os.path.join(root_dir, output_path)
+        apks = glob.glob(globpath)
+        if len(apks) > 1:
+            raise fdroidserver.exception.BuildException(
+                'Multiple apks match %s' % globpath, '\n'.join(apks)
+            )
+        if len(apks) < 1:
+            raise fdroidserver.exception.BuildException('No apks match %s' % globpath)
+        src = os.path.normpath(apks[0])
+    return src
+
+
+def check_build_success(app, build, p):
+    build_dir = fdroidserver.common.get_build_dir(app)
+
+    if os.path.isdir(os.path.join(build_dir, '.git')):
+        import git
+
+        commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
+    else:
+        commit_id = build.commit
+
+    if p is not None and p.returncode != 0:
+        raise fdroidserver.exception.BuildException(
+            "Build failed for %s:%s@%s" % (app.id, build.versionName, commit_id),
+            p.output,
+        )
+    logging.info(
+        "Successfully built version {versionName} of {appid} from {commit_id}".format(
+            versionName=build.versionName, appid=app.id, commit_id=commit_id
+        )
+    )
+
+
+def execute_postbuild(app, build, src):
+    root_dir = get_build_root_dir(app, build)
+    srclibpaths = get_srclibpaths(app, build)
+
+    if build.postbuild:
+        logging.info(f"Running 'postbuild' commands in {root_dir}")
+        cmd = fdroidserver.common.replace_config_vars("; ".join(build.postbuild), build)
+
+        # Substitute source library paths into commands...
+        for name, number, libpath in srclibpaths:
+            cmd = cmd.replace(f"$${name}$$", str(pathlib.Path.cwd() / libpath))
+
+        cmd = cmd.replace('$$OUT$$', str(pathlib.Path(src).resolve()))
+
+        p = fdroidserver.common.FDroidPopen(
+            ['bash', '-e', '-u', '-o', 'pipefail', '-x', '-c', cmd], cwd=root_dir
+        )
+
+        if p.returncode != 0:
+            raise fdroidserver.exception.BuildException(
+                "Error running postbuild command for " f"{app.id}:{build.versionName}",
+                p.output,
+            )
+
+
+def get_metadata_from_apk(app, build, apkfile):
+    """Get the required metadata from the built APK.
+
+    VersionName is allowed to be a blank string, i.e. ''
+
+    Parameters
+    ----------
+    app
+        The app metadata used to build the APK.
+    build
+        The build that resulted in the APK.
+    apkfile
+        The path of the APK file.
+
+    Returns
+    -------
+    versionCode
+        The versionCode from the APK or from the metadata is build.novcheck is
+        set.
+    versionName
+        The versionName from the APK or from the metadata is build.novcheck is
+        set.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.BuildException`
+        If native code should have been built but was not packaged, no version
+        information or no package ID could be found or there is a mismatch
+        between the package ID in the metadata and the one found in the APK.
+    """
+    appid, versionCode, versionName = fdroidserver.common.get_apk_id(apkfile)
+    native_code = fdroidserver.common.get_native_code(apkfile)
+
+    if build.buildjni and build.buildjni != ['no'] and not native_code:
+        raise fdroidserver.exception.BuildException(
+            "Native code should have been built but none was packaged"
+        )
+    if build.novcheck:
+        versionCode = build.versionCode
+        versionName = build.versionName
+    if not versionCode or versionName is None:
+        raise fdroidserver.exception.BuildException(
+            "Could not find version information in build in output"
+        )
+    if not appid:
+        raise fdroidserver.exception.BuildException(
+            "Could not find package ID in output"
+        )
+    if appid != app.id:
+        raise fdroidserver.exception.BuildException(
+            "Wrong package ID - build " + appid + " but expected " + app.id
+        )
+
+    return versionCode, versionName
+
+
+def validate_build_artifacts(app, build, src):
+    # Make sure it's not debuggable...
+    if fdroidserver.common.is_debuggable_or_testOnly(src):
+        raise fdroidserver.exception.BuildException(
+            "%s: debuggable or testOnly set in AndroidManifest.xml" % src
+        )
+
+    # By way of a sanity check, make sure the version and version
+    # code in our new APK match what we expect...
+    logging.debug("Checking " + src)
+    if not os.path.exists(src):
+        raise fdroidserver.exception.BuildException(
+            "Unsigned APK is not at expected location of " + src
+        )
+
+    if fdroidserver.common.get_file_extension(src) == 'apk':
+        vercode, version = get_metadata_from_apk(app, build, src)
+        if version != build.versionName or vercode != build.versionCode:
+            raise fdroidserver.exception.BuildException(
+                (
+                    "Unexpected version/version code in output;"
+                    " APK: '%s' / '%d', "
+                    " Expected: '%s' / '%d'"
+                )
+                % (version, vercode, build.versionName, build.versionCode)
+            )
+
+
+def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball=False):
+    tarname = fdroidserver.common.getsrcname(app, build)
+
+    # Copy the unsigned APK to our destination directory for further
+    # processing (by publish.py)...
+    dest = os.path.join(
+        output_dir,
+        fdroidserver.common.get_release_filename(
+            app, build, fdroidserver.common.get_file_extension(src)
+        ),
+    )
+    shutil.copyfile(src, dest)
+
+    # Move the source tarball into the output directory...
+    if output_dir != tmp_dir and not notarball:
+        shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname))
+
+
+def run_this_build(config, options, package_name, version_code):
+    """Run build for one specific version of an app localy.
+
+    :raises: various exceptions in case and of the pre-required conditions for the requested build are not met
+    """
+    app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
+
+    # not sure if this makes any sense to change open file limits since we know
+    # that this script will only ever build one app
+    rlimit_check()
+
+    logging.info(
+        "Building version %s (%s) of %s"
+        % (build.versionName, build.versionCode, app.id)
+    )
+
+    # init fdroid Popen wrapper
+    fdroidserver.common.set_FDroidPopen_env(build)
+    gradletasks = init_build(app, build, config)
+
+    sanitize_build_dir(app)
+
+    # this is where we'd call scanner.scan_source() in old build.py
+
+    # create tarball before building
+    # consider  this optional?
+    tmp_dir = pathlib.Path("./tmp")
+    tmp_dir.mkdir(exist_ok=True)
+    make_tarball(app, build, tmp_dir)
+
+    # Run a build command if one is required...
+    execute_build_commands(app, build)
+
+    # Build native stuff if required...
+    ndk_path = install_ndk(build, config)  # TODO: move to prepare step?
+    execute_buildjni_commands(app, build, ndk_path)
+
+    # Build the release...
+    p, bindir = execute_build(app, build, config, gradletasks)
+    check_build_success(app, build, p)
+    src = collect_build_output(app, build, p, bindir)
+
+    # Run a postbuild command if one is required...
+    execute_postbuild(app, build, src)
+
+    validate_build_artifacts(app, build, src)
+
+    # this is where we'd call scanner.scan_binary() in old build.py
+
+    move_build_output(app, build, src, tmp_dir)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=_(
+            "Build one specific in app. This command "
+            "assumes all required build tools are installed and "
+            "configured."
+        )
+    )
+    parser.add_argument(
+        "APP_VERSION",
+        help=_("app id and version code tuple (e.g. org.fdroid.fdroid:1019051)"),
+    )
+
+    # fdroid args/opts boilerplate
+    fdroidserver.common.setup_global_opts(parser)
+    options = fdroidserver.common.parse_args(parser)
+    config = fdroidserver.common.get_config()
+
+    try:
+        # read build target package name and version code from CLI arguments
+        package_name, version_code = fdroidserver.common.split_pkg_arg(
+            options.APP_VERSION
+        )
+        # trigger the build
+        run_this_build(config, options, package_name, version_code)
+    except Exception as e:
+        if options.verbose:
+            traceback.print_exc()
+        else:
+            print(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/build_local_sudo.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+#
+# build.py - part of the FDroid server tools
+# Copyright (C) 2024, Michael Pöhn <michael@poehn.at>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import logging
+import argparse
+
+from fdroidserver import _
+import fdroidserver.common
+
+
+def sudo_run(app, build):
+    # before doing anything, run the sudo commands to setup the VM
+    if build.sudo:
+        logging.info("Running 'sudo' commands in %s" % os.getcwd())
+
+        p = fdroidserver.common.FDroidPopen(
+            [
+                'sudo',
+                'DEBIAN_FRONTEND=noninteractive',
+                'bash',
+                '-e',
+                '-u',
+                '-o',
+                'pipefail',
+                '-x',
+                '-c',
+                '; '.join(build.sudo),
+            ]
+        )
+        if p.returncode != 0:
+            raise fdroidserver.exception.BuildException(
+                "Error running sudo command for %s:%s" % (app.id, build.versionName),
+                p.output,
+            )
+
+
+def sudo_lock_root(app, build):
+    p = fdroidserver.common.FDroidPopen(['sudo', 'passwd', '--lock', 'root'])
+    if p.returncode != 0:
+        raise fdroidserver.exception.BuildException(
+            "Error locking root account for %s:%s" % (app.id, build.versionName),
+            p.output,
+        )
+
+
+def sudo_uninstall(app, build):
+    p = fdroidserver.common.FDroidPopen(
+        ['sudo', 'SUDO_FORCE_REMOVE=yes', 'dpkg', '--purge', 'sudo']
+    )
+    if p.returncode != 0:
+        raise fdroidserver.exception.BuildException(
+            "Error removing sudo for %s:%s" % (app.id, build.versionName), p.output
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=_(
+            """Run sudo commands """
+        ),
+    )
+    parser.add_argument(
+        "APP_VERSION",
+        help=_("app id and version code tuple 'APPID:VERCODE'"),
+    )
+
+    # fdroid args/opts boilerplate
+    fdroidserver.common.setup_global_opts(parser)
+    options = fdroidserver.common.parse_args(parser)
+    config = fdroidserver.common.get_config()
+    config  # silcense pyflakes
+
+    package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION)
+    app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
+
+    # intialize FDroidPopen
+    # TODO: remove once FDroidPopen is replaced with vm/container exec
+    fdroidserver.common.set_FDroidPopen_env(build)
+
+    # run sudo stuff
+    sudo_run(app, build)
+    sudo_lock_root(app, build)
+    sudo_uninstall(app, build)
+
+
+if __name__ == "__main__":
+    main()

fdroidserver/common.py

@@ -870,6 +870,27 @@ def get_local_metadata_files():
     return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
 
 
+def split_pkg_arg(appid_versionCode_pair):
+    """Split 'appid:versionCode' pair into 2 separate values safely.
+
+    :raises ValueError: if argument is not parseable
+    :return: (appid, versionCode) tuple with the 2 parsed values
+    """
+    tokens = appid_versionCode_pair.split(":")
+    if len(tokens) != 2:
+        raise ValueError(
+            _("'{}' is not a valid pair of the form appId:versionCode pair").format(
+                appid_versionCode_pair
+            )
+        )
+    if not is_valid_package_name(tokens[0]):
+        raise ValueError(
+            _("'{}' does not start with a valid appId").format(appid_versionCode_pair)
+        )
+    versionCode = version_code_string_to_int(tokens[1])
+    return tokens[0], versionCode
+
+
 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
     """No summary.
 
@@ -1127,7 +1148,24 @@ def get_head_commit_id(git_repo):
 
 
 def setup_vcs(app):
-    """Checkout code from VCS and return instance of vcs and the build dir."""
+    """Create a VCS instance for given app.
+
+    This is a factory function that creates the correct type of VCS instance.
+    This doesn't checkout or clone any source code, it just creates a VCS
+    instance.
+
+    Parameters
+    ----------
+    app
+        metadata app object
+
+    Returns
+    -------
+    vcs
+        VCS instance corresponding to passed app
+    build_dir
+        source code checkout directory for the supplied app
+    """
     build_dir = get_build_dir(app)
 
     # Set up vcs interface and make sure we have the latest code...
@@ -2192,7 +2230,41 @@ def getsrclib(spec, srclib_dir, basepath=False,
     referencing it, which may be a subdirectory of the actual project. If
     you want the base directory of the project, pass 'basepath=True'.
 
-    spec and srclib_dir are both strings, not pathlib.Path.
+    Parameters
+    ----------
+    spec
+        srclib identifier (e.g. 'reproducible-apk-tools@v0.2.3').
+        must be string.
+    srclib_dir
+        base dir for holding checkouts of srclibs (usually './build/srclib').
+        must be a string.
+    basepath
+        changes the output of libdir to the base path, if set to True (default:
+        False)
+    raw
+        Don't sparese the spec instead use the unparsed spec as name, if set to
+        True (default: False)
+    prepare
+        Don't run `Prepare` commands in metadata, if set to False (default:
+        True)
+    preponly
+        Don't checkout the latest source code, if set to True (default: False)
+    refresh
+        Don't fetch latest source code from git remote, if set to False
+        (default: True)
+    build
+        metadata build object
+
+    Returns
+    -------
+    name
+        name of the srclib (e.g. 'mylib' when the spec is 'mylib@1.2.3')
+    number
+        number prefix from srclib spec (e.g. '7' when spec is '7:mylib@1.2.3')
+        (only used for ant builds)
+    libdir
+        (sub-)directory with the source code of this srclib (if basepath is set
+        this will ignore 'Subdir' from srclib metadata)
     """
     number = None
     subdir = None

fdroidserver/metadata.py

@@ -607,6 +607,45 @@ def read_metadata(appids={}, sort_by_time=False):
     return apps
 
 
+def read_build_metadata(package_name, version_code, check_disabled=True):
+    """Read 1 single metadata file from the file system + 1 singled out build.
+
+    Parameters
+    ----------
+    package_name
+        appid of the metadata supposed to be loaded (e.g. 'org.fdroid.fdroid')
+    version_code
+        android integer version identifier of a build (e.g. 1234)
+    check_disabled
+        If True this function will raise an exception in case the disabled flag
+        on the build or the metadata file is set.
+
+    Returns
+    -------
+    Tuple
+        A tuple of (metadata, build) dictionsaries, containing the pared data from
+        the metadata file.
+
+    Raises
+    ------
+    MetaDataException
+        If parsing or selecting the requested version fails.
+    """
+    m = read_metadata({package_name: version_code})
+    if len(m) != 1 or package_name not in m:
+        raise MetaDataException(f"Could not read metadata for '{package_name}:{version_code}' (metadata file might not be present or correct)")
+    if check_disabled and m[package_name].get("Disabled"):
+        raise MetaDataException(f"'{package_name}' disbaled in metadata file.")
+    if "Builds" not in m[package_name]:
+        raise MetaDataException(f"Cound not find 'Builds' section in '{package_name}' metadata file")
+    for b in m[package_name]["Builds"]:
+        if b['versionCode'] == version_code:
+            if check_disabled and b.get('disable'):
+                raise MetaDataException(f"'{package_name}:{version_code}' disabled in metadata file")
+            return m[package_name], b
+    raise MetaDataException(f"Could not find '{version_code}' in 'Builds' section of 'package_name' metadata file")
+
+
 def parse_metadata(metadatapath):
     """Parse metadata file, also checking the source repo for .fdroid.yml.
 

gradlew-fdroid

@@ -200,6 +200,7 @@ get_sha() {
         '8.5')    echo '9d926787066a081739e8200858338b4a69e837c3a821a33aca9db09dd4a41026' ;;
         '8.6')    echo '9631d53cf3e74bfa726893aee1f8994fee4e060c401335946dba2156f440f24c' ;;
         '8.7')    echo '544c35d6bd849ae8a5ed0bcea39ba677dc40f49df7d1835561582da2009b961d' ;;
+        '8.8')    echo 'a4b4158601f8636cdeeab09bd76afb640030bb5b144aafe261a5e8af027dc612' ;;
         *) exit 1
     esac
 }
@@ -220,7 +221,7 @@ d_gradle_plugin_ver_k=(8.4 8.3 8.2 8.1 8.0 7.4 7.3 7.2.0 7.1 7.0 4.2 4.1 4.0 3.6
 d_plugin_min_gradle_v=(8.6 8.4 8.2 8.0 8.0 7.5 7.4 7.3.3 7.2 7.0.2 6.7.1 6.5 6.1.1 5.6.4 5.4.1 5.1.1 4.10.1 4.6 4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
 
 # All gradle versions we know about
-plugin_v=(8.7 8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
+plugin_v=(8.8 8.7 8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
 
 v_all=${plugin_v[@]}