Merge branch 'build-refactoring' into 'master'

draft: build overhault

See merge request fdroid/fdroidserver!1486

fdroidserver/__main__.py

@@ -61,6 +61,7 @@ COMMANDS = OrderedDict([
 COMMANDS_INTERNAL = [
     "build_local_prepare",
     "build_local_run",
+    "build_local_sudo",
 ]
 
 

fdroidserver/build_local_run.py

@@ -47,7 +47,7 @@ def rlimit_check(apps_count=1):
 
 
 def install_ndk(build, config):
-    """Make sure the requested NDK version is or gets installed
+    """Make sure the requested NDK version is or gets installed.
 
     TODO: check if this should be moved to a script that runs before starting
     the build. e.g. `build_local_prepare` or `build_local_sudo`
@@ -57,6 +57,8 @@ def install_ndk(build, config):
     build
         Metadata build entry that's about the build and may contain the
         requested NDK version
+    config
+        dictonariy holding config/default data from `./config.yml`
     """
     ndk_path = build.ndk_path()
     if build.ndk or (build.buildjni and build.buildjni != ['no']):
@@ -89,14 +91,44 @@ def transform_first_char(string, method):
     return method(string[0]) + string[1:]
 
 
+def get_flavours_cmd(build):
+    """Get flavor string, preformatted for gradle cli.
+
+    Reads build flavors form metadata if any and reformats and concatenates
+    them to be ready for use as CLI arguments to gradle. This will treat the
+    vlue 'yes' as if there were not particular build flavor selected.
+
+    Parameters
+    ----------
+    build
+        The metadata build entry you'd like to read flavors from
+
+    Returns
+    -------
+    A string containing the build flavor for this build. If it's the default
+    flavor ("yes" in metadata) this returns an empty string. Returns None if
+    it's not a gradle build.
+    """
+    flavours = build.gradle
+
+    if flavours == ['yes']:
+        flavours = []
+
+    flavours_cmd = ''.join(
+        [transform_first_char(flav, str.upper) for flav in flavours]
+    )
+
+    return flavours_cmd
+
+
 def init_build(app, build, config):
-    # We need to clean via the build tool in case the binary dirs are
-    # different from the default ones
     root_dir = get_build_root_dir(app, build)
 
     p = None
     gradletasks = []
-    flavours_cmd = None
+
+    # We need to clean via the build tool in case the binary dirs are
+    # different from the default ones
 
     bmethod = build.build_method()
     if bmethod == 'maven':
@@ -118,13 +150,7 @@ def init_build(app, build, config):
         if build.preassemble:
             gradletasks += build.preassemble
 
-        flavours = build.gradle
+        flavours_cmd = get_flavours_cmd(build)
-        if flavours == ['yes']:
-            flavours = []
-
-        flavours_cmd = ''.join(
-            [transform_first_char(flav, str.upper) for flav in flavours]
-        )
 
         gradletasks += ['assemble' + flavours_cmd + 'Release']
 
@@ -151,15 +177,16 @@ def init_build(app, build, config):
             "Error cleaning %s:%s" % (app.id, build.versionName), p.output
         )
 
-    return gradletasks, flavours_cmd
+    return gradletasks
 
 
 def sanitize_build_dir(app):
-    """delete build output directories
+    """Delete build output directories.
 
     This function deletes the default build/binary/target/... output
     directories for follwoing build tools: gradle, maven, ant, jni. It also
-    deletes gradle-wrapper if present.
+    deletes gradle-wrapper if present. It just uses parths, hardcoded here,
+    it doesn't call and build system clean routines.
 
     Parameters
     ----------
@@ -235,8 +262,21 @@ def make_tarball(app, build, tmp_dir):
     tarball.close()
 
 
-def execute_build_commands(app, build, srclibpaths):
+def execute_build_commands(app, build):
+    """Execute `bulid` commands if present in metadata.
+
+    see: https://f-droid.org/docs/Build_Metadata_Reference/#build_build
+
+    Parameters
+    ----------
+    app
+        metadata app object
+    build
+        metadata build object
+    """
     root_dir = get_build_root_dir(app, build)
+    srclibpaths = get_srclibpaths(app, build)
+
     if build.build:
         logging.info("Running 'build' commands in %s" % root_dir)
         cmd = fdroidserver.common.replace_config_vars("; ".join(build.build), build)
@@ -256,18 +296,40 @@ def execute_build_commands(app, build, srclibpaths):
             )
 
 
-def checkout_srclibs(app, build):
+def get_srclibpaths(app, build):
-    """"""
+    """Get srclibpaths list of tuples.
-    build_dir = fdroidserver.common.get_build_dir(app)
+
+    This will just assemble the srclibpaths list of tuples, it won't fetch
+    or checkout any source code, identical to return value of
+    common.prepare_souce().
+
+    Parameters
+    ----------
+    app
+        metadata app object
+    build
+        metadata build object
+
+    Returns
+    -------
+    List of srclibpath tuples
+    """
+    vcs, _ = fdroidserver.common.setup_vcs(app)
+
     srclibpaths = []
     if build.srclibs:
         logging.info("Collecting source libraries")
         for lib in build.srclibs:
             srclibpaths.append(
                 fdroidserver.common.getsrclib(
-                    lib, build_dir, prepare=False, refresh=False, build=build
+                    lib, "./build/srclib", prepare=False, refresh=False, build=build
                 )
             )
+
+    basesrclib = vcs.getsrclib()
+    if basesrclib:
+        srclibpaths.append(basesrclib)
+
     return srclibpaths
 
 
@@ -304,7 +366,6 @@ def execute_buildjni_commands(app, build, ndk_path):
 
 
 def execute_build(app, build, config, gradletasks):
-    build_dir = fdroidserver.common.get_build_dir(app)
     root_dir = get_build_root_dir(app, build)
 
     p = None
@@ -377,7 +438,7 @@ def execute_build(app, build, config, gradletasks):
     return p, bindir
 
 
-def collect_build_output(app, build, p, flavours_cmd, bindir):
+def collect_build_output(app, build, p, bindir):
     root_dir = get_build_root_dir(app, build)
 
     omethod = build.output_method()
@@ -432,6 +493,7 @@ def collect_build_output(app, build, p, flavours_cmd, bindir):
         # If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
         # a subdirectory corresponding to the flavour command used, but with different
         # capitalization.
+        flavours_cmd = get_flavours_cmd(build)
         if flavours_cmd:
             apk_dirs.append(
                 os.path.join(
@@ -507,8 +569,9 @@ def check_build_success(app, build, p):
     )
 
 
-def execute_postbuild(app, build, src, srclibpaths):
+def execute_postbuild(app, build, src):
     root_dir = get_build_root_dir(app, build)
+    srclibpaths = get_srclibpaths(app, build)
 
     if build.postbuild:
         logging.info(f"Running 'postbuild' commands in {root_dir}")
@@ -634,11 +697,10 @@ def move_build_output(app, build, src, tmp_dir, output_dir="unsigned", notarball
 
 
 def run_this_build(config, options, package_name, version_code):
-    """run build for one specific version of an app localy
+    """Run build for one specific version of an app localy.
 
     :raises: various exceptions in case and of the pre-required conditions for the requested build are not met
     """
-
     app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
 
     # not sure if this makes any sense to change open file limits since we know
@@ -652,7 +714,7 @@ def run_this_build(config, options, package_name, version_code):
 
     # init fdroid Popen wrapper
     fdroidserver.common.set_FDroidPopen_env(build)
-    gradletasks, flavours_cmd = init_build(app, build, config)
+    gradletasks = init_build(app, build, config)
 
     sanitize_build_dir(app)
 
@@ -665,8 +727,7 @@ def run_this_build(config, options, package_name, version_code):
     make_tarball(app, build, tmp_dir)
 
     # Run a build command if one is required...
-    srclibpaths = checkout_srclibs(app, build)
+    execute_build_commands(app, build)
-    execute_build_commands(app, build, srclibpaths)
 
     # Build native stuff if required...
     ndk_path = install_ndk(build, config)  # TODO: move to prepare step?
@@ -675,10 +736,10 @@ def run_this_build(config, options, package_name, version_code):
     # Build the release...
     p, bindir = execute_build(app, build, config, gradletasks)
     check_build_success(app, build, p)
-    src = collect_build_output(app, build, p, flavours_cmd, bindir)
+    src = collect_build_output(app, build, p, bindir)
 
     # Run a postbuild command if one is required...
-    execute_postbuild(app, build, src, srclibpaths)
+    execute_postbuild(app, build, src)
 
     validate_build_artifacts(app, build, src)
 

fdroidserver/build_local_sudo.py

@@ -90,7 +90,6 @@ def main():
     package_name, version_code = fdroidserver.common.split_pkg_arg(options.APP_VERSION)
     app, build = fdroidserver.metadata.read_build_metadata(package_name, version_code)
 
-
     # intialize FDroidPopen
     # TODO: remove once FDroidPopen is replaced with vm/container exec
     fdroidserver.common.set_FDroidPopen_env(build)
@@ -100,5 +99,6 @@ def main():
     sudo_lock_root(app, build)
     sudo_uninstall(app, build)
 
+
 if __name__ == "__main__":
     main()

fdroidserver/common.py

@@ -1148,7 +1148,24 @@ def get_head_commit_id(git_repo):
 
 
 def setup_vcs(app):
-    """Checkout code from VCS and return instance of vcs and the build dir."""
+    """Create a VCS instance for given app.
+
+    This is a factory function that creates the correct type of VCS instance.
+    This doesn't checkout or clone any source code, it just creates a VCS
+    instance.
+
+    Parameters
+    ----------
+    app
+        metadata app object
+
+    Returns
+    -------
+    vcs
+        VCS instance corresponding to passed app
+    build_dir
+        source code checkout directory for the supplied app
+    """
     build_dir = get_build_dir(app)
 
     # Set up vcs interface and make sure we have the latest code...
@@ -2213,7 +2230,41 @@ def getsrclib(spec, srclib_dir, basepath=False,
     referencing it, which may be a subdirectory of the actual project. If
     you want the base directory of the project, pass 'basepath=True'.
 
-    spec and srclib_dir are both strings, not pathlib.Path.
+    Parameters
+    ----------
+    spec
+        srclib identifier (e.g. 'reproducible-apk-tools@v0.2.3').
+        must be string.
+    srclib_dir
+        base dir for holding checkouts of srclibs (usually './build/srclib').
+        must be a string.
+    basepath
+        changes the output of libdir to the base path, if set to True (default:
+        False)
+    raw
+        Don't sparese the spec instead use the unparsed spec as name, if set to
+        True (default: False)
+    prepare
+        Don't run `Prepare` commands in metadata, if set to False (default:
+        True)
+    preponly
+        Don't checkout the latest source code, if set to True (default: False)
+    refresh
+        Don't fetch latest source code from git remote, if set to False
+        (default: True)
+    build
+        metadata build object
+
+    Returns
+    -------
+    name
+        name of the srclib (e.g. 'mylib' when the spec is 'mylib@1.2.3')
+    number
+        number prefix from srclib spec (e.g. '7' when spec is '7:mylib@1.2.3')
+        (only used for ant builds)
+    libdir
+        (sub-)directory with the source code of this srclib (if basepath is set
+        this will ignore 'Subdir' from srclib metadata)
     """
     number = None
     subdir = None

fdroidserver/metadata.py

@@ -608,7 +608,7 @@ def read_metadata(appids={}, sort_by_time=False):
 
 
 def read_build_metadata(package_name, version_code, check_disabled=True):
-    """read 1 single metadata file from the file system + 1 singled out build
+    """Read 1 single metadata file from the file system + 1 singled out build.
 
     Parameters
     ----------

gradlew-fdroid

@@ -200,6 +200,7 @@ get_sha() {
         '8.5')    echo '9d926787066a081739e8200858338b4a69e837c3a821a33aca9db09dd4a41026' ;;
         '8.6')    echo '9631d53cf3e74bfa726893aee1f8994fee4e060c401335946dba2156f440f24c' ;;
         '8.7')    echo '544c35d6bd849ae8a5ed0bcea39ba677dc40f49df7d1835561582da2009b961d' ;;
+        '8.8')    echo 'a4b4158601f8636cdeeab09bd76afb640030bb5b144aafe261a5e8af027dc612' ;;
         *) exit 1
     esac
 }
@@ -220,7 +221,7 @@ d_gradle_plugin_ver_k=(8.4 8.3 8.2 8.1 8.0 7.4 7.3 7.2.0 7.1 7.0 4.2 4.1 4.0 3.6
 d_plugin_min_gradle_v=(8.6 8.4 8.2 8.0 8.0 7.5 7.4 7.3.3 7.2 7.0.2 6.7.1 6.5 6.1.1 5.6.4 5.4.1 5.1.1 4.10.1 4.6 4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
 
 # All gradle versions we know about
-plugin_v=(8.7 8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
+plugin_v=(8.8 8.7 8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
 
 v_all=${plugin_v[@]}
 

tests/build_local_run.TestCase

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-# import json
-# import shutil
-# import logging
-import inspect
-# import tempfile
-import unittest
-import unittest.mock
-
-localmodule = os.path.realpath(
-    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
-)
-print('localmodule: ' + localmodule)
-if localmodule not in sys.path:
-    sys.path.insert(0, localmodule)
-
-import fdroidserver.build_local_run
-import fdroidserver.metadata
-
-
-class GetBuildRootDirTest(unittest.TestCase):
-    def test_no_subdir(self):
-        app = unittest.mock.MagicMock()
-        app.id = "fake.app"
-        build = unittest.mock.MagicMock()
-        build.subdir = None
-
-        gbdmock = unittest.mock.Mock(return_value="ok")
-        with unittest.mock.patch("fdroidserver.common.get_build_dir", gbdmock):
-            r = fdroidserver.build_local_run.get_build_root_dir(app, build)
-
-        gbdmock.assert_called_once_with(app)
-        self.assertEqual(r, "ok")
-
-    def test_subdir(self):
-        app = unittest.mock.MagicMock()
-        app.id = "fake.app"
-        build = unittest.mock.MagicMock()
-        build.subdir = "abc/def"
-
-        gbdmock = unittest.mock.Mock(return_value="ok")
-        with unittest.mock.patch("fdroidserver.common.get_build_dir", gbdmock):
-            r = fdroidserver.build_local_run.get_build_root_dir(app, build)
-
-        gbdmock.assert_called_once_with(app)
-        self.assertEqual(r, "ok/abc/def")
-
-
-if __name__ == "__main__":
-    os.chdir(os.path.dirname(__file__))
-
-    import argparse
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        default=False,
-        help="Spew out even more information than normal",
-    )
-    fdroidserver.common.options = fdroidserver.common.parse_args(parser)
-
-    newSuite = unittest.TestSuite()
-    newSuite.addTest(unittest.makeSuite(GetBuildRootDirTest))
-    unittest.main(failfast=False)