Merge branch 'docstrings' into 'master'

docstrings and code format from !1436

See merge request fdroid/fdroidserver!1482

fdroidserver/btlog.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+"""Update the binary transparency log for a URL."""
 #
 # btlog.py - part of the FDroid server tools
 # Copyright (C) 2017, Hans-Christoph Steiner <hans@eds.org>
@@ -26,7 +27,6 @@
 # client app so its not easy for the server to distinguish this from
 # the F-Droid client.
 
-
 import collections
 import defusedxml.minidom
 import git
@@ -39,6 +39,7 @@ import shutil
 import tempfile
 import zipfile
 from argparse import ArgumentParser
+from typing import Optional
 
 from . import _
 from . import common
@@ -50,14 +51,30 @@ options = None
 
 
 def make_binary_transparency_log(
-    repodirs, btrepo='binary_transparency', url=None, commit_title='fdroid update'
+    repodirs: collections.abc.Iterable,
+    btrepo: str = 'binary_transparency',
+    url: Optional[str] = None,
+    commit_title: str = 'fdroid update',
 ):
     """Log the indexes in a standalone git repo to serve as a "binary transparency" log.
 
-    References
+    Parameters
     ----------
-    https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
+    repodirs
+        The directories of the F-Droid repository to generate the binary
+        transparency log for.
+    btrepo
+        The path to the Git repository of the binary transparency log.
+    url
+        The URL of the F-Droid repository to generate the binary transparency
+        log for.
+    commit_title
+        The commit title for commits in the binary transparency log Git
+        repository.
 
+    Notes
+    -----
+    Also see https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies .
     """
     logging.info('Committing indexes to ' + btrepo)
     if os.path.exists(os.path.join(btrepo, '.git')):
@@ -149,6 +166,16 @@ For more info on this idea:
 
 
 def main():
+    """Generate or update a binary transparency log for a F-Droid repository.
+
+    The behaviour of this function is influenced by the configuration file as
+    well as command line parameters.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.FDroidException`
+        If the specified or default Git repository does not exist.
+    """
     global options
 
     parser = ArgumentParser()

fdroidserver/build.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+"""Build a package from source."""
 #
 # build.py - part of the FDroid server tools
 # Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
@@ -59,15 +60,30 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
     Parameters
     ----------
     app
-        app metadata dict
+        The metadata of the app to build.
     build
+        The build of the app to build.
     vcs
-        version control system controller object
+        The version control system controller object of the app.
     build_dir
-        local source-code checkout of app
+        The local source-code checkout directory of the app.
     output_dir
-        target folder for the build result
+        The target folder for the build result.
+    log_dir
+        The directory in the VM where the build logs are getting stored.
     force
+        Don't refresh the already cloned repository and make the build stop on
+        exceptions.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.BuildException`
+        If Paramiko is not installed, a srclib directory or srclib metadata
+        file is unexpectedly missing, the build process in the VM failed or
+        output files of the build process are missing.
+    :exc:`~fdroidserver.exception.FDroidException`
+        If the Buildserver ID could not be obtained or copying a directory to
+        the server failed.
     """
     global buildserverid, ssh_channel
 
@@ -115,8 +131,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         # Put all the necessary files in place...
         ftp.chdir(homedir)
 
-        # Helper to copy the contents of a directory to the server...
         def send_dir(path):
+            """Copy the contents of a directory to the server."""
             logging.debug("rsyncing %s to %s" % (path, ftp.getcwd()))
             # TODO this should move to `vagrant rsync` from >= v1.5
             try:
@@ -316,6 +332,15 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
 
 def force_gradle_build_tools(build_dir, build_tools):
+    """Manipulate build tools version used in top level gradle file.
+
+    Parameters
+    ----------
+    build_dir
+        The directory to start looking for gradle files.
+    build_tools
+        The build tools version that should be forced to use.
+    """
     for root, dirs, files in os.walk(build_dir):
         for filename in files:
             if not filename.endswith('.gradle'):
@@ -342,6 +367,31 @@ def get_metadata_from_apk(app, build, apkfile):
     """Get the required metadata from the built APK.
 
     VersionName is allowed to be a blank string, i.e. ''
+
+    Parameters
+    ----------
+    app
+        The app metadata used to build the APK.
+    build
+        The build that resulted in the APK.
+    apkfile
+        The path of the APK file.
+
+    Returns
+    -------
+    versionCode
+        The versionCode from the APK or from the metadata is build.novcheck is
+        set.
+    versionName
+        The versionName from the APK or from the metadata is build.novcheck is
+        set.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.BuildException`
+        If native code should have been built but was not packaged, no version
+        information or no package ID could be found or there is a mismatch
+        between the package ID in the metadata and the one found in the APK.
     """
     appid, versionCode, versionName = common.get_apk_id(apkfile)
     native_code = common.get_native_code(apkfile)
@@ -362,7 +412,56 @@ def get_metadata_from_apk(app, build, apkfile):
 
 
 def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
-    """Do a build locally."""
+    """Do a build locally.
+
+    Parameters
+    ----------
+    app
+        The metadata of the app to build.
+    build
+        The build of the app to build.
+    vcs
+        The version control system controller object of the app.
+    build_dir
+        The local source-code checkout directory of the app.
+    output_dir
+        The target folder for the build result.
+    log_dir
+        The directory in the VM where the build logs are getting stored.
+    srclib_dir
+        The path to the srclibs directory, usually 'build/srclib'.
+    extlib_dir
+        The path to the extlibs directory, usually 'build/extlib'.
+    tmp_dir
+        The temporary directory for building the source tarball.
+    force
+        Don't refresh the already cloned repository and make the build stop on
+        exceptions.
+    onserver
+        Assume the build is happening inside the VM.
+    refresh
+        Enable fetching the latest refs from the VCS remote.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.BuildException`
+        If running a `sudo` command failed, locking the root account failed,
+        `sudo` couldn't be removed, cleaning the build environment failed,
+        skipping the scanning has been requested but `scandelete` is present,
+        errors occurred during scanning, running the `build` commands from the
+        metadata failed, building native code failed, building with the
+        specified build method failed, no output could be found with build
+        method `maven`, more or less than one APK were found with build method
+        `gradle`, less or more than one APKs match the `output` glob specified
+        in the metadata, running a `postbuild` command specified in the
+        metadata failed, the built APK is debuggable, the unsigned APK is not
+        at the expected location, the APK does not contain the expected
+        `versionName` and `versionCode` or undesired package names have been
+        found in the APK.
+    :exc:`~fdroidserver.exception.FDroidException`
+        If no Android NDK version could be found and the build isn't run in a
+        builder VM, the selected Android NDK is not a directory.
+    """
     ndk_path = build.ndk_path()
     if build.ndk or (build.buildjni and build.buildjni != ['no']):
         if not ndk_path:
@@ -766,23 +865,47 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
 
     Parameters
     ----------
+    app
+        The metadata of the app to build.
+    build
+        The build of the app to build.
+    build_dir
+        The local source-code checkout directory of the app.
     output_dir
-        The directory where the build output will go.
+        The directory where the build output will go.  Usually this is the
-        Usually this is the 'unsigned' directory.
+        'unsigned' directory.
+    log_dir
+        The directory in the VM where the build logs are getting stored.
+    also_check_dir
+        An additional location for checking if the build is necessary (usually
+        the archive repo).
+    srclib_dir
+        The path to the srclibs directory, usually 'build/srclib'.
+    extlib_dir
+        The path to the extlibs directory, usually 'build/extlib'.
+    tmp_dir
+        The temporary directory for building the source tarball of the app to
+        build.
     repo_dir
         The repo directory - used for checking if the build is necessary.
-    also_check_dir
+    vcs
-        An additional location for checking if the build
+        The version control system controller object of the app to build.
-        is necessary (usually the archive repo)
     test
-        True if building in test mode, in which case the build will
+        True if building in test mode, in which case the build will always
-        always happen, even if the output already exists. In test mode, the
+        happen, even if the output already exists.  In test mode, the output
-        output directory should be a temporary location, not any of the real
+        directory should be a temporary location, not any of the real ones.
-        ones.
+    server
+        Use buildserver VM for building.
+    force
+        Build app regardless of disabled state or scanner errors.
+    onserver
+        Assume the build is happening inside the VM.
+    refresh
+        Enable fetching the latest refs from the VCS remote.
 
     Returns
     -------
-    Boolean
+    status
         True if the build was done, False if it wasn't necessary.
     """
     dest_file = common.get_release_filename(app, build)
@@ -821,7 +944,13 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
 
 
 def force_halt_build(timeout):
-    """Halt the currently running Vagrant VM, to be called from a Timer."""
+    """Halt the currently running Vagrant VM, to be called from a Timer.
+
+    Parameters
+    ----------
+    timeout
+        The timeout in seconds.
+    """
     logging.error(_('Force halting build after {0} sec timeout!').format(timeout))
     timeout_event.set()
     if ssh_channel:
@@ -845,7 +974,9 @@ def parse_commandline():
     Returns
     -------
     options
+        The resulting options parsed from the command line arguments.
     parser
+        The argument parser.
     """
     parser = argparse.ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
@@ -905,6 +1036,22 @@ timeout_event = threading.Event()
 
 
 def main():
+    """Build a package from source.
+
+    The behaviour of this function is influenced by the configuration file as
+    well as command line parameters.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.FDroidException`
+        If more than one local metadata file has been found, no app metadata
+        has been found, there are no apps to process, downloading binaries for
+        checking the reproducibility of a built binary failed, the built binary
+        is different from supplied reference binary, the reference binary is
+        signed with a different signing key than expected, a VCS error occured
+        while building an app or a different error occured while building an
+        app.
+    """
     global options, config, buildserverid, fdroidserverid
 
     options, parser = parse_commandline()

fdroidserver/checkupdates.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+"""Check for updates to applications."""
 #
 # checkupdates.py - part of the FDroid server tools
 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com

fdroidserver/import_subcommand.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+"""Extract application metadata from a source repository."""
 #
 # import_subcommand.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@@ -30,6 +31,7 @@ import yaml
 from argparse import ArgumentParser
 import logging
 from pathlib import Path
+from typing import Optional
 
 try:
     from yaml import CSafeLoader as SafeLoader
@@ -53,7 +55,19 @@ def handle_retree_error_on_windows(function, path, excinfo):
         function(path)
 
 
-def clone_to_tmp_dir(app):
+def clone_to_tmp_dir(app: metadata.App) -> Path:
+    """Clone the source repository of an app to a temporary directory for further processing.
+
+    Parameters
+    ----------
+    app
+        The App instance to clone the source of.
+
+    Returns
+    -------
+    tmp_dir
+        The (temporary) directory the apps source has been cloned into.
+    """
     tmp_dir = Path('tmp')
     tmp_dir.mkdir(exist_ok=True)
 
@@ -67,7 +81,7 @@ def clone_to_tmp_dir(app):
     return tmp_dir
 
 
-def getrepofrompage(url):
+def getrepofrompage(url: str) -> tuple[Optional[str], str]:
     """Get the repo type and address from the given web page.
 
     The page is scanned in a rather naive manner for 'git clone xxxx',
@@ -75,6 +89,17 @@ def getrepofrompage(url):
     that's the information we want.  Returns repotype, address, or
     None, reason
 
+    Parameters
+    ----------
+    url
+        The url to look for repository information at.
+
+    Returns
+    -------
+    repotype_or_none
+        The found repository type or None if an error occured.
+    address_or_reason
+        The address to the found repository or the reason if an error occured.
     """
     if not url.startswith('http'):
         return (None, _('{url} does not start with "http"!'.format(url=url)))
@@ -120,13 +145,29 @@ def getrepofrompage(url):
     return (None, _("No information found.") + page)
 
 
-def get_app_from_url(url):
+def get_app_from_url(url: str) -> metadata.App:
     """Guess basic app metadata from the URL.
 
     The URL must include a network hostname, unless it is an lp:,
     file:, or git/ssh URL.  This throws ValueError on bad URLs to
     match urlparse().
 
+    Parameters
+    ----------
+    url
+        The URL to look to look for app metadata at.
+
+    Returns
+    -------
+    app
+        App instance with the found metadata.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.FDroidException`
+        If the VCS type could not be determined.
+    :exc:`ValueError`
+        If the URL is invalid.
     """
     parsed = urllib.parse.urlparse(url)
     invalid_url = False
@@ -183,6 +224,19 @@ def get_app_from_url(url):
 
 
 def main():
+    """Extract app metadata and write it to a file.
+
+    The behaviour of this function is influenced by the configuration file as
+    well as command line parameters.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.FDroidException`
+        If the repository already has local metadata, no URL is specified and
+        the current directory is not a Git repository, no application ID could
+        be found, no Gradle project could be found or there is already metadata
+        for the found application ID.
+    """
     global config, options
 
     # Parse command line...

fdroidserver/init.py

@@ -282,13 +282,13 @@ def main():
     msg += '\n\n'
     msg += (
         _(
-            '''To complete the setup, add your APKs to "%s"
+            """To complete the setup, add your APKs to "%s"
 then run "fdroid update -c; fdroid update".  You might also want to edit
 "config.yml" to set the URL, repo name, and more.  You should also set up
 a signing key (a temporary one might have been automatically generated).
 
 For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo
-and https://f-droid.org/docs/Signing_Process'''
+and https://f-droid.org/docs/Signing_Process"""
         )
         % os.path.join(fdroiddir, 'repo')
     )

fdroidserver/lint.py

@@ -214,7 +214,7 @@ regex_checks = {
             _("Forbidden HTML tags"),
         ),
         (
-            re.compile(r'''.*\s+src=["']javascript:.*'''),
+            re.compile(r""".*\s+src=["']javascript:.*"""),
             _("Javascript in HTML src attributes"),
         ),
     ],
@@ -459,9 +459,12 @@ def check_builds(app):
                             "Branch '{branch}' used as commit in srclib '{srclib}'"
                         ).format(branch=s, srclib=srclib)
                 else:
-                    yield _(
+                    yield (
-                        'srclibs missing name and/or @'
+                        _('srclibs missing name and/or @')
-                    ) + ' (srclibs: ' + srclib + ')'
+                        + ' (srclibs: '
+                        + srclib
+                        + ')'
+                    )
         for key in build.keys():
             if key not in supported_flags:
                 yield _('%s is not an accepted build field') % key

fdroidserver/nightly.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+"""Set up an app build for a nightly build repo."""
 #
 # nightly.py - part of the FDroid server tools
 # Copyright (C) 2017 Hans-Christoph Steiner <hans@eds.org>
@@ -32,6 +33,7 @@ import tempfile
 import yaml
 from urllib.parse import urlparse
 from argparse import ArgumentParser
+from typing import Optional
 
 from . import _
 from . import common
@@ -48,12 +50,38 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US'
 NIGHTLY = '-nightly'
 
 
-def _get_keystore_secret_var(keystore):
+def _get_keystore_secret_var(keystore: str) -> str:
+    """Get keystore secret as base64.
+
+    Parameters
+    ----------
+    keystore
+        The path of the keystore.
+
+    Returns
+    -------
+    base64_secret
+        The keystore secret as base64 string.
+    """
     with open(keystore, 'rb') as fp:
         return base64.standard_b64encode(fp.read()).decode('ascii')
 
 
-def _ssh_key_from_debug_keystore(keystore=None):
+def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str:
+    """Convert a debug keystore to an SSH private key.
+
+    This leaves the original keystore file in place.
+
+    Parameters
+    ----------
+    keystore
+        The keystore to convert to a SSH private key.
+
+    Returns
+    -------
+    key_path
+        The SSH private key file path in the temporary directory.
+    """
     if keystore is None:
         # set this here so it can be overridden in the tests
         # TODO convert this to a class to get rid of this nonsense
@@ -148,7 +176,23 @@ def _ssh_key_from_debug_keystore(keystore=None):
     return ssh_private_key_file
 
 
-def get_repo_base_url(clone_url, repo_git_base, force_type=None):
+def get_repo_base_url(clone_url: str, repo_git_base: str, force_type: Optional[str] = None) -> str:
+    """Generate the base URL for the F-Droid repository.
+
+    Parameters
+    ----------
+    clone_url
+        The URL to clone the Git repository.
+    repo_git_base
+        The project path of the Git repository at the Git forge.
+    force_type
+        The Git forge of the project.
+
+    Returns
+    -------
+    repo_base_url
+        The base URL of the F-Droid repository.
+    """
     if force_type is None:
         force_type = urlparse(clone_url).netloc
     if force_type == 'gitlab.com':
@@ -160,6 +204,17 @@ def get_repo_base_url(clone_url, repo_git_base, force_type=None):
 
 
 def main():
+    """Deploy to F-Droid repository or generate SSH private key from keystore.
+
+    The behaviour of this function is influenced by the configuration file as
+    well as command line parameters.
+
+    Raises
+    ------
+    :exc:`~fdroidserver.exception.VCSException`
+        If the nightly Git repository could not be cloned during an attempt to
+        deploy.
+    """
     parser = ArgumentParser()
     common.setup_global_opts(parser)
     parser.add_argument(