From 6c27fec94b560d5ea9bf3b7dd87c9be422e4c912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20D=C3=BCster?= Date: Sat, 28 Oct 2023 08:32:26 +0200 Subject: [PATCH 1/6] [import] Add more docstrings --- fdroidserver/import_subcommand.py | 60 +++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/fdroidserver/import_subcommand.py b/fdroidserver/import_subcommand.py index 16c00f5b..d1105914 100644 --- a/fdroidserver/import_subcommand.py +++ b/fdroidserver/import_subcommand.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""Extract application metadata from a source repository.""" # # import_subcommand.py - part of the FDroid server tools # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com @@ -30,6 +31,7 @@ import yaml from argparse import ArgumentParser import logging from pathlib import Path +from typing import Optional try: from yaml import CSafeLoader as SafeLoader @@ -53,7 +55,19 @@ def handle_retree_error_on_windows(function, path, excinfo): function(path) -def clone_to_tmp_dir(app): +def clone_to_tmp_dir(app: metadata.App) -> Path: + """Clone the source repository of an app to a temporary directory for further processing. + + Parameters + ---------- + app + The App instance to clone the source of. + + Returns + ------- + tmp_dir + The (temporary) directory the apps source has been cloned into. + """ tmp_dir = Path('tmp') tmp_dir.mkdir(exist_ok=True) @@ -67,7 +81,7 @@ def clone_to_tmp_dir(app): return tmp_dir -def getrepofrompage(url): +def getrepofrompage(url: str) -> tuple[Optional[str], str]: """Get the repo type and address from the given web page. The page is scanned in a rather naive manner for 'git clone xxxx', @@ -75,6 +89,17 @@ def getrepofrompage(url): that's the information we want. Returns repotype, address, or None, reason + Parameters + ---------- + url + The url to look for repository information at. + + Returns + ------- + repotype_or_none + The found repository type or None if an error occured. + address_or_reason + The address to the found repository or the reason if an error occured. """ if not url.startswith('http'): return (None, _('{url} does not start with "http"!'.format(url=url))) @@ -120,13 +145,29 @@ def getrepofrompage(url): return (None, _("No information found.") + page) -def get_app_from_url(url): +def get_app_from_url(url: str) -> metadata.App: """Guess basic app metadata from the URL. The URL must include a network hostname, unless it is an lp:, file:, or git/ssh URL. This throws ValueError on bad URLs to match urlparse(). + Parameters + ---------- + url + The URL to look to look for app metadata at. + + Returns + ------- + app + App instance with the found metadata. + + Raises + ------ + :exc:`~fdroidserver.exception.FDroidException` + If the VCS type could not be determined. + :exc:`ValueError` + If the URL is invalid. """ parsed = urllib.parse.urlparse(url) invalid_url = False @@ -183,6 +224,19 @@ def get_app_from_url(url): def main(): + """Extract app metadata and write it to a file. + + The behaviour of this function is influenced by the configuration file as + well as command line parameters. + + Raises + ------ + :exc:`~fdroidserver.exception.FDroidException` + If the repository already has local metadata, no URL is specified and + the current directory is not a Git repository, no application ID could + be found, no Gradle project could be found or there is already metadata + for the found application ID. + """ global config, options # Parse command line... From 97346a2cba788b20ffbfb98ab8c301fa25968e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20D=C3=BCster?= Date: Sun, 29 Oct 2023 14:10:30 +0100 Subject: [PATCH 2/6] [nightly] Add more docstrings --- fdroidserver/nightly.py | 61 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 400ad0ba..9471a26e 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""Set up an app build for a nightly build repo.""" # # nightly.py - part of the FDroid server tools # Copyright (C) 2017 Hans-Christoph Steiner @@ -32,6 +33,7 @@ import tempfile import yaml from urllib.parse import urlparse from argparse import ArgumentParser +from typing import Optional from . import _ from . import common @@ -48,12 +50,38 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US' NIGHTLY = '-nightly' -def _get_keystore_secret_var(keystore): +def _get_keystore_secret_var(keystore: str) -> str: + """Get keystore secret as base64. + + Parameters + ---------- + keystore + The path of the keystore. + + Returns + ------- + base64_secret + The keystore secret as base64 string. + """ with open(keystore, 'rb') as fp: return base64.standard_b64encode(fp.read()).decode('ascii') -def _ssh_key_from_debug_keystore(keystore=None): +def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str: + """Convert a debug keystore to an SSH private key. + + This leaves the original keystore file in place. + + Parameters + ---------- + keystore + The keystore to convert to a SSH private key. + + Returns + ------- + key_path + The SSH private key file path in the temporary directory. + """ if keystore is None: # set this here so it can be overridden in the tests # TODO convert this to a class to get rid of this nonsense @@ -148,7 +176,23 @@ def _ssh_key_from_debug_keystore(keystore=None): return ssh_private_key_file -def get_repo_base_url(clone_url, repo_git_base, force_type=None): +def get_repo_base_url(clone_url: str, repo_git_base: str, force_type: Optional[str] = None) -> str: + """Generate the base URL for the F-Droid repository. + + Parameters + ---------- + clone_url + The URL to clone the Git repository. + repo_git_base + The project path of the Git repository at the Git forge. + force_type + The Git forge of the project. + + Returns + ------- + repo_base_url + The base URL of the F-Droid repository. + """ if force_type is None: force_type = urlparse(clone_url).netloc if force_type == 'gitlab.com': @@ -160,6 +204,17 @@ def get_repo_base_url(clone_url, repo_git_base, force_type=None): def main(): + """Deploy to F-Droid repository or generate SSH private key from keystore. + + The behaviour of this function is influenced by the configuration file as + well as command line parameters. + + Raises + ------ + :exc:`~fdroidserver.exception.VCSException` + If the nightly Git repository could not be cloned during an attempt to + deploy. + """ parser = ArgumentParser() common.setup_global_opts(parser) parser.add_argument( From 4109e8fb035064953a718d35cc2f0b990a668cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20D=C3=BCster?= Date: Sun, 29 Oct 2023 15:15:29 +0100 Subject: [PATCH 3/6] [checkupdates] Add module docstring --- fdroidserver/checkupdates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index aa41e5b7..07a4d668 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""Check for updates to applications.""" # # checkupdates.py - part of the FDroid server tools # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com From 1c707589409a32bc1725244917138db06e9ac458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20D=C3=BCster?= Date: Mon, 30 Oct 2023 08:57:21 +0100 Subject: [PATCH 4/6] [btlog] Add more docstrings --- fdroidserver/btlog.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py index ea79bc8a..fc49ac9a 100755 --- a/fdroidserver/btlog.py +++ b/fdroidserver/btlog.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""Update the binary transparency log for a URL.""" # # btlog.py - part of the FDroid server tools # Copyright (C) 2017, Hans-Christoph Steiner @@ -39,6 +40,7 @@ import shutil import tempfile import zipfile from argparse import ArgumentParser +from typing import Optional from . import _ from . import common @@ -50,14 +52,30 @@ options = None def make_binary_transparency_log( - repodirs, btrepo='binary_transparency', url=None, commit_title='fdroid update' + repodirs: collections.abc.Iterable, + btrepo: str = 'binary_transparency', + url: Optional[str] = None, + commit_title: str = 'fdroid update', ): """Log the indexes in a standalone git repo to serve as a "binary transparency" log. - References + Parameters ---------- - https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies + repodirs + The directories of the F-Droid repository to generate the binary + transparency log for. + btrepo + The path to the Git repository of the binary transparency log. + url + The URL of the F-Droid repository to generate the binary transparency + log for. + commit_title + The commit title for commits in the binary transparency log Git + repository. + Notes + ----- + Also see https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies . """ logging.info('Committing indexes to ' + btrepo) if os.path.exists(os.path.join(btrepo, '.git')): @@ -149,6 +167,16 @@ For more info on this idea: def main(): + """Generate or update a binary transparency log for a F-Droid repository. + + The behaviour of this function is influenced by the configuration file as + well as command line parameters. + + Raises + ------ + :exc:`~fdroidserver.exception.FDroidException` + If the specified or default Git repository does not exist. + """ global options parser = ArgumentParser() From df27405d8b87be5c38ef0853fbba2eb1bed7e49c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20D=C3=BCster?= Date: Wed, 24 Jan 2024 13:08:01 +0100 Subject: [PATCH 5/6] [build] Add more docstrings --- fdroidserver/build.py | 181 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 164 insertions(+), 17 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 84f19ed5..02e52315 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +"""Build a package from source.""" # # build.py - part of the FDroid server tools # Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com @@ -59,15 +60,30 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): Parameters ---------- app - app metadata dict + The metadata of the app to build. build + The build of the app to build. vcs - version control system controller object + The version control system controller object of the app. build_dir - local source-code checkout of app + The local source-code checkout directory of the app. output_dir - target folder for the build result + The target folder for the build result. + log_dir + The directory in the VM where the build logs are getting stored. force + Don't refresh the already cloned repository and make the build stop on + exceptions. + + Raises + ------ + :exc:`~fdroidserver.exception.BuildException` + If Paramiko is not installed, a srclib directory or srclib metadata + file is unexpectedly missing, the build process in the VM failed or + output files of the build process are missing. + :exc:`~fdroidserver.exception.FDroidException` + If the Buildserver ID could not be obtained or copying a directory to + the server failed. """ global buildserverid, ssh_channel @@ -115,8 +131,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): # Put all the necessary files in place... ftp.chdir(homedir) - # Helper to copy the contents of a directory to the server... def send_dir(path): + """Copy the contents of a directory to the server.""" logging.debug("rsyncing %s to %s" % (path, ftp.getcwd())) # TODO this should move to `vagrant rsync` from >= v1.5 try: @@ -316,6 +332,15 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): def force_gradle_build_tools(build_dir, build_tools): + """Manipulate build tools version used in top level gradle file. + + Parameters + ---------- + build_dir + The directory to start looking for gradle files. + build_tools + The build tools version that should be forced to use. + """ for root, dirs, files in os.walk(build_dir): for filename in files: if not filename.endswith('.gradle'): @@ -342,6 +367,31 @@ def get_metadata_from_apk(app, build, apkfile): """Get the required metadata from the built APK. VersionName is allowed to be a blank string, i.e. '' + + Parameters + ---------- + app + The app metadata used to build the APK. + build + The build that resulted in the APK. + apkfile + The path of the APK file. + + Returns + ------- + versionCode + The versionCode from the APK or from the metadata is build.novcheck is + set. + versionName + The versionName from the APK or from the metadata is build.novcheck is + set. + + Raises + ------ + :exc:`~fdroidserver.exception.BuildException` + If native code should have been built but was not packaged, no version + information or no package ID could be found or there is a mismatch + between the package ID in the metadata and the one found in the APK. """ appid, versionCode, versionName = common.get_apk_id(apkfile) native_code = common.get_native_code(apkfile) @@ -362,7 +412,56 @@ def get_metadata_from_apk(app, build, apkfile): def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): - """Do a build locally.""" + """Do a build locally. + + Parameters + ---------- + app + The metadata of the app to build. + build + The build of the app to build. + vcs + The version control system controller object of the app. + build_dir + The local source-code checkout directory of the app. + output_dir + The target folder for the build result. + log_dir + The directory in the VM where the build logs are getting stored. + srclib_dir + The path to the srclibs directory, usually 'build/srclib'. + extlib_dir + The path to the extlibs directory, usually 'build/extlib'. + tmp_dir + The temporary directory for building the source tarball. + force + Don't refresh the already cloned repository and make the build stop on + exceptions. + onserver + Assume the build is happening inside the VM. + refresh + Enable fetching the latest refs from the VCS remote. + + Raises + ------ + :exc:`~fdroidserver.exception.BuildException` + If running a `sudo` command failed, locking the root account failed, + `sudo` couldn't be removed, cleaning the build environment failed, + skipping the scanning has been requested but `scandelete` is present, + errors occurred during scanning, running the `build` commands from the + metadata failed, building native code failed, building with the + specified build method failed, no output could be found with build + method `maven`, more or less than one APK were found with build method + `gradle`, less or more than one APKs match the `output` glob specified + in the metadata, running a `postbuild` command specified in the + metadata failed, the built APK is debuggable, the unsigned APK is not + at the expected location, the APK does not contain the expected + `versionName` and `versionCode` or undesired package names have been + found in the APK. + :exc:`~fdroidserver.exception.FDroidException` + If no Android NDK version could be found and the build isn't run in a + builder VM, the selected Android NDK is not a directory. + """ ndk_path = build.ndk_path() if build.ndk or (build.buildjni and build.buildjni != ['no']): if not ndk_path: @@ -766,23 +865,47 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, Parameters ---------- + app + The metadata of the app to build. + build + The build of the app to build. + build_dir + The local source-code checkout directory of the app. output_dir - The directory where the build output will go. - Usually this is the 'unsigned' directory. + The directory where the build output will go. Usually this is the + 'unsigned' directory. + log_dir + The directory in the VM where the build logs are getting stored. + also_check_dir + An additional location for checking if the build is necessary (usually + the archive repo). + srclib_dir + The path to the srclibs directory, usually 'build/srclib'. + extlib_dir + The path to the extlibs directory, usually 'build/extlib'. + tmp_dir + The temporary directory for building the source tarball of the app to + build. repo_dir The repo directory - used for checking if the build is necessary. - also_check_dir - An additional location for checking if the build - is necessary (usually the archive repo) + vcs + The version control system controller object of the app to build. test - True if building in test mode, in which case the build will - always happen, even if the output already exists. In test mode, the - output directory should be a temporary location, not any of the real - ones. + True if building in test mode, in which case the build will always + happen, even if the output already exists. In test mode, the output + directory should be a temporary location, not any of the real ones. + server + Use buildserver VM for building. + force + Build app regardless of disabled state or scanner errors. + onserver + Assume the build is happening inside the VM. + refresh + Enable fetching the latest refs from the VCS remote. Returns ------- - Boolean + status True if the build was done, False if it wasn't necessary. """ dest_file = common.get_release_filename(app, build) @@ -821,7 +944,13 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, def force_halt_build(timeout): - """Halt the currently running Vagrant VM, to be called from a Timer.""" + """Halt the currently running Vagrant VM, to be called from a Timer. + + Parameters + ---------- + timeout + The timeout in seconds. + """ logging.error(_('Force halting build after {0} sec timeout!').format(timeout)) timeout_event.set() if ssh_channel: @@ -845,7 +974,9 @@ def parse_commandline(): Returns ------- options + The resulting options parsed from the command line arguments. parser + The argument parser. """ parser = argparse.ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) @@ -905,6 +1036,22 @@ timeout_event = threading.Event() def main(): + """Build a package from source. + + The behaviour of this function is influenced by the configuration file as + well as command line parameters. + + Raises + ------ + :exc:`~fdroidserver.exception.FDroidException` + If more than one local metadata file has been found, no app metadata + has been found, there are no apps to process, downloading binaries for + checking the reproducibility of a built binary failed, the built binary + is different from supplied reference binary, the reference binary is + signed with a different signing key than expected, a VCS error occured + while building an app or a different error occured while building an + app. + """ global options, config, buildserverid, fdroidserverid options, parser = parse_commandline() From dbdefe200c8ce8d6474f7ae97be1005f06c32ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20D=C3=BCster?= Date: Fri, 26 Jan 2024 08:08:37 +0100 Subject: [PATCH 6/6] Format files with ruff --- fdroidserver/btlog.py | 1 - fdroidserver/init.py | 4 ++-- fdroidserver/lint.py | 11 +++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py index fc49ac9a..3154b3de 100755 --- a/fdroidserver/btlog.py +++ b/fdroidserver/btlog.py @@ -27,7 +27,6 @@ # client app so its not easy for the server to distinguish this from # the F-Droid client. - import collections import defusedxml.minidom import git diff --git a/fdroidserver/init.py b/fdroidserver/init.py index d3195288..ab5a3e48 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -282,13 +282,13 @@ def main(): msg += '\n\n' msg += ( _( - '''To complete the setup, add your APKs to "%s" + """To complete the setup, add your APKs to "%s" then run "fdroid update -c; fdroid update". You might also want to edit "config.yml" to set the URL, repo name, and more. You should also set up a signing key (a temporary one might have been automatically generated). For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo -and https://f-droid.org/docs/Signing_Process''' +and https://f-droid.org/docs/Signing_Process""" ) % os.path.join(fdroiddir, 'repo') ) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 351667ba..d7a9783d 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -214,7 +214,7 @@ regex_checks = { _("Forbidden HTML tags"), ), ( - re.compile(r'''.*\s+src=["']javascript:.*'''), + re.compile(r""".*\s+src=["']javascript:.*"""), _("Javascript in HTML src attributes"), ), ], @@ -459,9 +459,12 @@ def check_builds(app): "Branch '{branch}' used as commit in srclib '{srclib}'" ).format(branch=s, srclib=srclib) else: - yield _( - 'srclibs missing name and/or @' - ) + ' (srclibs: ' + srclib + ')' + yield ( + _('srclibs missing name and/or @') + + ' (srclibs: ' + + srclib + + ')' + ) for key in build.keys(): if key not in supported_flags: yield _('%s is not an accepted build field') % key