diff --git a/.gitignore b/.gitignore index c192310b..223807b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -/config.py -/makebs.config.py *~ *.pyc *.class @@ -28,9 +26,11 @@ tmp/ # files used in manual testing /config.py +/config.yml /tmp/ /logs/ /metadata/ +/makebs.config.py makebuildserver.config.py /tests/.fdroid.keypass.txt /tests/.fdroid.keystorepass.txt diff --git a/MANIFEST.in b/MANIFEST.in index b3dcf59b..9f0d5f5d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include buildserver/config.buildserver.py +include buildserver/config.buildserver.yml include buildserver/provision-android-ndk include buildserver/provision-android-sdk include buildserver/provision-apt-get-install @@ -8,7 +8,7 @@ include buildserver/setup-env-vars include buildserver/Vagrantfile include CHANGELOG.md include completion/bash-completion -include examples/config.py +include examples/config.yml include examples/fdroid-icon.png include examples/makebuildserver.config.py include examples/opensc-fdroid.cfg diff --git a/buildserver/config.buildserver.py b/buildserver/config.buildserver.py deleted file mode 100644 index e09188ec..00000000 --- a/buildserver/config.buildserver.py +++ /dev/null @@ -1,19 +0,0 @@ -sdk_path = "/home/vagrant/android-sdk" -ndk_paths = { - 'r10e': "/home/vagrant/android-ndk/r10e", - 'r11c': "/home/vagrant/android-ndk/r11c", - 'r12b': "/home/vagrant/android-ndk/r12b", - 'r13b': "/home/vagrant/android-ndk/r13b", - 'r14b': "/home/vagrant/android-ndk/r14b", - 'r15c': "/home/vagrant/android-ndk/r15c", - 'r16b': "/home/vagrant/android-ndk/r16b", - 'r17c': "/home/vagrant/android-ndk/r17c", - 'r18b': "/home/vagrant/android-ndk/r18b", - 'r19c': "/home/vagrant/android-ndk/r19c", - 'r20b': "/home/vagrant/android-ndk/r20b", - 'r21d': "/home/vagrant/android-ndk/r21d", -} -java_paths = { - '8': "/usr/lib/jvm/java-8-openjdk-amd64", -} -gradle_version_dir = "/opt/gradle/versions" diff --git a/buildserver/config.buildserver.yml b/buildserver/config.buildserver.yml new file mode 100644 index 00000000..b94fc601 --- /dev/null +++ b/buildserver/config.buildserver.yml @@ -0,0 +1,19 @@ +sdk_path: /home/vagrant/android-sdk +ndk_paths: + r10e: /home/vagrant/android-ndk/r10e + r11c: /home/vagrant/android-ndk/r11c + r12b: /home/vagrant/android-ndk/r12b + r13b: /home/vagrant/android-ndk/r13b + r14b: /home/vagrant/android-ndk/r14b + r15c: /home/vagrant/android-ndk/r15c + r16b: /home/vagrant/android-ndk/r16b + r17c: /home/vagrant/android-ndk/r17c + r18b: /home/vagrant/android-ndk/r18b + r19c: /home/vagrant/android-ndk/r19c + r20b: /home/vagrant/android-ndk/r20b + r21d: /home/vagrant/android-ndk/r21d + +java_paths: + 8: /usr/lib/jvm/java-8-openjdk-amd64 + +gradle_version_dir: /opt/gradle/versions diff --git a/examples/config.py b/examples/config.yml similarity index 73% rename from examples/config.py rename to examples/config.yml index c9c1cbc1..555699fd 100644 --- a/examples/config.py +++ b/examples/config.yml @@ -1,82 +1,78 @@ -#!/usr/bin/env python3 - -# Copy this file to config.py, then amend the settings below according to +--- +# Copy this file to config.yml, then amend the settings below according to # your system configuration. # Custom path to the Android SDK, defaults to $ANDROID_HOME -# sdk_path = "$ANDROID_HOME" +# sdk_path: $ANDROID_HOME # Custom paths to various versions of the Android NDK, defaults to 'r12b' set # to $ANDROID_NDK. Most users will have the latest at $ANDROID_NDK, which is # used by default. If a version is missing or assigned to None, it is assumed # not installed. -# ndk_paths = { -# 'r10e': None, -# 'r11c': None, -# 'r12b': "$ANDROID_NDK", -# 'r13b': None, -# 'r14b': None, -# 'r15c': None, -# 'r16b': None, -# 'r17c': None, -# 'r18b': None, -# 'r19c': None, -# 'r20b': None, -# 'r21d': None, -# } +# ndk_paths: +# r10e: None +# r11c: None +# r12b: $ANDROID_NDK +# r13b: None +# r14b: None +# r15c: None +# r16b: None +# r17c: None +# r18b: None +# r19c: None +# r20b: None +# r21d: None # Directory to store downloaded tools in (i.e. gradle versions) # By default, these are stored in ~/.cache/fdroidserver -# cachedir = cache +# cachedir: cache -# java_paths = { -# '8': "/usr/lib/jvm/java-8-openjdk", -# } +# Specify paths to each major Java release that you want to support +# java_paths: +# 8: /usr/lib/jvm/java-8-openjdk # Build tools version to be used -# build_tools = "28.0.3" +# build_tools: 28.0.3 # Command or path to binary for running Ant -# ant = "ant" +# ant: ant # Command or path to binary for running maven 3 -# mvn3 = "mvn" +# mvn3: mvn # Command or path to binary for running Gradle # Defaults to using an internal gradle wrapper (gradlew-fdroid). -# gradle = "gradle" +# gradle: gradle # Always scan the APKs produced by `fdroid build` for known non-free classes -# scan_binary = True +# scan_binary: true # Set the maximum age (in days) of an index that a client should accept from # this repo. Setting it to 0 or not setting it at all disables this # functionality. If you do set this to a non-zero value, you need to ensure # that your index is updated much more frequently than the specified interval. # The same policy is applied to the archive repo, if there is one. -# repo_maxage = 0 +# repo_maxage: 0 -repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo" -repo_name = "My First F-Droid Repo Demo" -repo_icon = "fdroid-icon.png" -repo_description = """ -This is a repository of apps to be used with F-Droid. Applications in this -repository are either official binaries built by the original application -developers, or are binaries built from source by the admin of f-droid.org -using the tools on https://gitlab.com/u/fdroid. -""" +repo_url: https://MyFirstFDroidRepo.org/fdroid/repo +repo_name: My First F-Droid Repo Demo +repo_icon: fdroid-icon.png +repo_description: | + This is a repository of apps to be used with F-Droid. Applications in this + repository are either official binaries built by the original application + developers, or are binaries built from source by the admin of f-droid.org + using the tools on https://gitlab.com/u/fdroid. # As above, but for the archive repo. # archive_older sets the number of versions kept in the main repo, with all # older ones going to the archive. Set it to 0, and there will be no archive # repository, and no need to define the other archive_ values. -archive_older = 3 -archive_url = "https://f-droid.org/archive" -archive_name = "My First F-Droid Archive Demo" -archive_icon = "fdroid-icon.png" -archive_description = """ -The repository of older versions of applications from the main demo repository. -""" +archive_older: 3 +archive_url: https://f-droid.org/archive +archive_name: My First F-Droid Archive Demo +archive_icon: fdroid-icon.png +archive_description: | + The repository of older versions of packages from the main demo repository. # This allows a specific kind of insecure APK to be included in the # 'repo' section. Since April 2017, APK signatures that use MD5 are @@ -85,77 +81,82 @@ The repository of older versions of applications from the main demo repository. # disabled signatures to the archive. This option stops that # behavior, and lets those APKs stay part of 'repo'. # -# allow_disabled_algorithms = True +# allow_disabled_algorithms: true # Normally, all apps are collected into a single app repository, like on # https://f-droid.org. For certain situations, it is better to make a repo # that is made up of APKs only from a single app. For example, an automated # build server that publishes nightly builds. -# per_app_repos = True +# per_app_repos: true # `fdroid update` will create a link to the current version of a given app. # This provides a static path to the current APK. To disable the creation of # this link, uncomment this: -# make_current_version_link = False +# make_current_version_link: false # By default, the "current version" link will be based on the "Name" of the # app from the metadata. You can change it to use a different field from the # metadata here: -# current_version_name_source = 'packageName' +# current_version_name_source: packageName # Optionally, override home directory for gpg -# gpghome = '/home/fdroid/somewhere/else/.gnupg' +# gpghome: /home/fdroid/somewhere/else/.gnupg # The ID of a GPG key for making detached signatures for apks. Optional. -# gpgkey = '1DBA2E89' +# gpgkey: 1DBA2E89 # The key (from the keystore defined below) to be used for signing the # repository itself. This is the same name you would give to keytool or # jarsigner using -alias. (Not needed in an unsigned repository). -# repo_keyalias = "fdroidrepo" +# repo_keyalias: fdroidrepo # Optionally, the public key for the key defined by repo_keyalias above can # be specified here. There is no need to do this, as the public key can and # will be retrieved from the keystore when needed. However, specifying it # manually can allow some processing to take place without access to the # keystore. -# repo_pubkey = "..." +# repo_pubkey: ... # The keystore to use for release keys when building. This needs to be # somewhere safe and secure, and backed up! The best way to manage these # sensitive keys is to use a "smartcard" (aka Hardware Security Module). To # configure F-Droid to use a smartcard, set the keystore file using the keyword -# "NONE" (i.e. keystore = "NONE"). That makes Java find the keystore on the +# "NONE" (i.e. keystore: "NONE"). That makes Java find the keystore on the # smartcard based on 'smartcardoptions' below. -# keystore = "~/.local/share/fdroidserver/keystore.jks" +# keystore: ~/.local/share/fdroidserver/keystore.jks # You should not need to change these at all, unless you have a very # customized setup for using smartcards in Java with keytool/jarsigner -# smartcardoptions = "-storetype PKCS11 -providerName SunPKCS11-OpenSC \ -# -providerClass sun.security.pkcs11.SunPKCS11 \ -# -providerArg opensc-fdroid.cfg" +# smartcardoptions: | +# -storetype PKCS11 -providerName SunPKCS11-OpenSC +# -providerClass sun.security.pkcs11.SunPKCS11 +# -providerArg opensc-fdroid.cfg # The password for the keystore (at least 6 characters). If this password is # different than the keypass below, it can be OK to store the password in this # file for real use. But in general, sensitive passwords should not be stored # in text files! -# keystorepass = "password1" +# keystorepass: password1 # The password for keys - the same is used for each auto-generated key as well # as for the repository key. You should not normally store this password in a # file since it is a sensitive password. -# keypass = "password2" +# keypass: password2 # The distinguished name used for all keys. -# keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US" +# keydname: CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US # Use this to override the auto-generated key aliases with specific ones # for particular applications. Normally, just leave it empty. -# keyaliases = {} -# keyaliases['com.example.app'] = 'example' +# +# keyaliases: +# com.example.app: example +# # You can also force an app to use the same key alias as another one, using # the @ prefix. -# keyaliases['com.example.another.plugin'] = '@com.example.another' +# +# keyaliases: +# com.example.another.plugin: "@com.example.another" # The full path to the root of the repository. It must be specified in @@ -166,11 +167,11 @@ The repository of older versions of applications from the main demo repository. # multiple servers to sync to by wrapping the whole thing in {} or [], and # including the serverwebroot strings in a comma-separated list. # -# serverwebroot = 'user@example:/var/www/fdroid' -# serverwebroot = { -# 'foo.com:/usr/share/nginx/www/fdroid', -# 'bar.info:/var/www/fdroid', -# } +# serverwebroot: user@example:/var/www/fdroid +# serverwebroot: +# - foo.com:/usr/share/nginx/www/fdroid +# - bar.info:/var/www/fdroid + # When running fdroid processes on a remote server, it is possible to # publish extra information about the status. Each fdroid sub-command @@ -181,25 +182,24 @@ The repository of older versions of applications from the main demo repository. # .../repo/$APPID_$VERCODE.log.gz. These files are also pushed to all # servers configured in 'serverwebroot'. # -# deploy_process_logs = True +# deploy_process_logs: true # The full URL to a git remote repository. You can include # multiple servers to mirror to by wrapping the whole thing in {} or [], and # including the servergitmirrors strings in a comma-separated list. # Servers listed here will also be automatically inserted in the mirrors list. # -# servergitmirrors = 'https://github.com/user/repo' -# servergitmirrors = { -# 'https://github.com/user/repo', -# 'https://gitlab.com/user/repo', -# } +# servergitmirrors: https://github.com/user/repo +# servergitmirrors: +# - https://github.com/user/repo +# - https://gitlab.com/user/repo # Most git hosting services have hard size limits for each git repo. # `fdroid deploy` will delete the git history when the git mirror repo # approaches this limit to ensure that the repo will still fit when # pushed. GitHub recommends 1GB, gitlab.com recommends 10GB. # -# git_mirror_size_limit = '10GB' +# git_mirror_size_limit: 10GB # Any mirrors of this repo, for example all of the servers declared in # serverwebroot and all the servers declared in servergitmirrors, @@ -210,14 +210,13 @@ The repository of older versions of applications from the main demo repository. # and the archive, if it is enabled. So these URLs should end in the # 'fdroid' base of the F-Droid part of the web server like serverwebroot. # -# mirrors = ( -# 'https://foo.bar/fdroid', -# 'http://foobarfoobarfoobar.onion/fdroid', -# ) +# mirrors: +# - https://foo.bar/fdroid +# - http://foobarfoobarfoobar.onion/fdroid # optionally specify which identity file to use when using rsync or git over SSH # -# identity_file = '~/.ssh/fdroid_id_rsa' +# identity_file: ~/.ssh/fdroid_id_rsa # If you are running the repo signing process on a completely offline machine, @@ -228,7 +227,7 @@ The repository of older versions of applications from the main demo repository. # standard folder called 'fdroid' as the specified folder is recommended, like # with serverwebroot. # -# local_copy_dir = '/media/MyUSBThumbDrive/fdroid' +# local_copy_dir: /media/MyUSBThumbDrive/fdroid # If you are using local_copy_dir on an offline build/signing server, once the @@ -236,19 +235,21 @@ The repository of older versions of applications from the main demo repository. # synced to the copy on the online machine. To make that happen # automatically, set sync_from_local_copy_dir to True: # -# sync_from_local_copy_dir = True +# sync_from_local_copy_dir: true # To upload the repo to an Amazon S3 bucket using `fdroid server # update`. Warning, this deletes and recreates the whole fdroid/ # directory each time. This prefers s3cmd, but can also use # apache-libcloud. To customize how s3cmd interacts with the cloud -# provider, create a 's3cfg' file next to this file (config.py), and +# provider, create a 's3cfg' file next to this file (config.yml), and # those settings will be used instead of any 'aws' variable below. +# Secrets can be fetched from environment variables to ensure that +# they are not leaked as part of this file. # -# awsbucket = 'myawsfdroid' -# awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA' -# awssecretkey = 'yourverysecretkeywordpassphraserighthere' +# awsbucket: myawsfdroid +# awsaccesskeyid: SEE0CHAITHEIMAUR2USA +# awssecretkey: {env: awssecretkey} # If you want to force 'fdroid server' to use a non-standard serverwebroot. @@ -256,69 +257,72 @@ The repository of older versions of applications from the main demo repository. # '/fdroid'. (Please note that some client features expect repository URLs # to end in '/fdroid/repo'.) # -# nonstandardwebroot = False +# nonstandardwebroot: false # If you want to upload the release apk file to androidobservatory.org # -# androidobservatory = False +# androidobservatory: false # If you want to upload the release apk file to virustotal.com # You have to enter your profile apikey to enable the upload. # -# virustotal_apikey = "virustotal_apikey" +# virustotal_apikey: 9872987234982734 +# +# Or get it from an environment variable: +# +# virustotal_apikey: {env: virustotal_apikey} # The build logs can be posted to a mediawiki instance, like on f-droid.org. -# wiki_protocol = "http" -# wiki_server = "server" -# wiki_path = "/wiki/" -# wiki_user = "login" -# wiki_password = "1234" +# wiki_protocol: http +# wiki_server: server +# wiki_path: /wiki/ +# wiki_user: login +# wiki_password: 1234 # Keep a log of all generated index files in a git repo to provide a # "binary transparency" log for anyone to check the history of the # binaries that are published. This is in the form of a "git remote", # which this machine where `fdroid update` is run has already been # configured to allow push access (e.g. ssh key, username/password, etc) -# binary_transparency_remote = "git@gitlab.com:fdroid/binary-transparency-log.git" +# binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git # Only set this to true when running a repository where you want to generate # stats, and only then on the master build servers, not a development # machine. If you want to keep the "added" and "last updated" dates for each # app and APK in your repo, then you should enable this. -# update_stats = True +# update_stats: true # When used with stats, this is a list of IP addresses that are ignored for # calculation purposes. -# stats_ignore = [] +# stats_ignore: [] # Server stats logs are retrieved from. Required when update_stats is True. -# stats_server = "example.com" +# stats_server: example.com # User stats logs are retrieved from. Required when update_stats is True. -# stats_user = "bob" +# stats_user: bob # Use the following to push stats to a Carbon instance: -# stats_to_carbon = False -# carbon_host = '0.0.0.0' -# carbon_port = 2003 +# stats_to_carbon: false +# carbon_host: 0.0.0.0 +# carbon_port: 2003 # Set this to true to always use a build server. This saves specifying the # --server option on dedicated secure build server hosts. -# build_server_always = True +# build_server_always: true # Limit in number of characters that fields can take up # Only the fields listed here are supported, defaults shown -# char_limits = { -# 'author': 256, -# 'name': 50, -# 'summary': 80, -# 'description': 4000, -# 'video': 256, -# 'whatsNew': 500, -# } +# char_limits: +# author: 256 +# name: 50 +# summary: 80 +# description: 4000 +# video: 256 +# whatsNew: 500 # It is possible for the server operator to specify lists of apps that # must be installed or uninstalled on the client (aka "push installs). @@ -327,16 +331,14 @@ The repository of older versions of applications from the main demo repository. # the packageNames listed. This is protected by the same signing key # as the app index metadata. # -# install_list = ( -# 'at.bitfire.davdroid', -# 'com.fsck.k9', -# 'us.replicant', -# ) +# install_list: +# - at.bitfire.davdroid +# - com.fsck.k9 +# - us.replicant # -# uninstall_list = ( -# 'com.facebook.orca', -# 'com.android.vending', -# ) +# uninstall_list: +# - com.facebook.orca +# - com.android.vending # `fdroid lint` checks licenses in metadata against a built white list. By # default we will require license metadata to be present and only allow @@ -350,7 +352,6 @@ The repository of older versions of applications from the main demo repository. # Just supply a custom list of licene names you would like to allow. Setting # this to `None` disables this lint check. # -# lint_licenses = ( -# 'Custom-License-A', -# 'Another-License', -# ) +# lint_licenses: +# - Custom-License-A +# - Another-License diff --git a/examples/template.yml b/examples/template.yml index 72d584aa..c9e565f6 100644 --- a/examples/template.yml +++ b/examples/template.yml @@ -6,7 +6,7 @@ Donate: null License: Unknown Categories: -- Internet + - Internet IssueTracker: '' SourceCode: '' @@ -15,7 +15,7 @@ Changelog: '' Name: . Summary: . Description: | - . + . ArchivePolicy: 2 versions -RequiresRoot: No +RequiresRoot: false diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 02d4e5d6..79be829d 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -141,8 +141,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.chdir(homedir) ftp.put(os.path.join(serverpath, '..', 'buildserver', - 'config.buildserver.py'), 'config.py') - ftp.chmod('config.py', 0o600) + 'config.buildserver.yml'), 'config.yml') + ftp.chmod('config.yml', 0o600) # Copy over the ID (head commit hash) of the fdroidserver in use... with open(os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'), 'wb') as fp: diff --git a/fdroidserver/common.py b/fdroidserver/common.py index df9ffe25..aa522da3 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -39,6 +39,7 @@ import socket import base64 import urllib.parse import urllib.request +import yaml import zipfile import tempfile import json @@ -284,14 +285,18 @@ def regsub_file(pattern, repl, path): f.write(text) -def read_config(opts, config_file='config.py'): +def read_config(opts): """Read the repository config The config is read from config_file, which is in the current directory when any of the repo management commands are used. If - there is a local metadata file in the git repo, then config.py is + there is a local metadata file in the git repo, then the config is not required, just use defaults. + config.yml is the preferred form because no code is executed when + reading it. config.py is deprecated and supported for backwards + compatibility. + """ global config, options @@ -301,14 +306,25 @@ def read_config(opts, config_file='config.py'): options = opts config = {} + config_file = 'config.yml' + old_config_file = 'config.py' - if os.path.isfile(config_file): + if os.path.exists(config_file) and os.path.exists(old_config_file): + logging.error(_("""Conflicting config files! Using {newfile}, ignoring {oldfile}!""") + .format(oldfile=old_config_file, newfile=config_file)) + + if os.path.exists(config_file): logging.debug(_("Reading '{config_file}'").format(config_file=config_file)) - with io.open(config_file, "rb") as f: - code = compile(f.read(), config_file, 'exec') - exec(code, None, config) # nosec TODO switch to YAML file + with open(config_file) as fp: + config = yaml.safe_load(fp) + elif os.path.exists(old_config_file): + logging.warning(_("""{oldfile} is deprecated, use {newfile}""") + .format(oldfile=old_config_file, newfile=config_file)) + with io.open(old_config_file, "rb") as fp: + code = compile(fp.read(), old_config_file, 'exec') + exec(code, None, config) # nosec TODO automatically migrate else: - logging.warning(_("No 'config.py' found, using defaults.")) + logging.warning(_("No config.yml found, using defaults.")) for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'): if k in config: @@ -329,10 +345,14 @@ def read_config(opts, config_file='config.py'): '-providerArg', 'opensc-fdroid.cfg'] if any(k in config for k in ["keystore", "keystorepass", "keypass"]): - st = os.stat(config_file) + if os.path.exists(config_file): + f = config_file + elif os.path.exists(old_config_file): + f = old_config_file + st = os.stat(f) if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO: logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!") - .format(config_file=config_file)) + .format(config_file=f)) fill_config_defaults(config) @@ -368,6 +388,29 @@ def read_config(opts, config_file='config.py'): limit = config['git_mirror_size_limit'] config['git_mirror_size_limit'] = parse_human_readable_size(limit) + for configname, dictvalue in config.items(): + if configname == 'java_paths': + new = dict() + for k, v in dictvalue.items(): + new[str(k)] = v + config[configname] = new + elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases'): + continue + elif isinstance(dictvalue, dict): + for k, v in dictvalue.items(): + if k == 'env': + env = os.getenv(v) + if env: + config[configname] = env + else: + del(config[configname]) + logging.error(_('Environment variable {var} from {configname} is not set!') + .format(var=k, configname=configname)) + else: + del(config[configname]) + logging.error(_('Unknown entry {key} in {configname}') + .format(key=k, configname=configname)) + return config @@ -396,10 +439,10 @@ def assert_config_keystore(config): nosigningkey = False if 'repo_keyalias' not in config: nosigningkey = True - logging.critical(_("'repo_keyalias' not found in config.py!")) + logging.critical(_("'repo_keyalias' not found in config.yml!")) if 'keystore' not in config: nosigningkey = True - logging.critical(_("'keystore' not found in config.py!")) + logging.critical(_("'keystore' not found in config.yml!")) elif config['keystore'] == 'NONE': if not config.get('smartcardoptions'): nosigningkey = True @@ -409,10 +452,10 @@ def assert_config_keystore(config): logging.critical("'" + config['keystore'] + "' does not exist!") if 'keystorepass' not in config: nosigningkey = True - logging.critical(_("'keystorepass' not found in config.py!")) + logging.critical(_("'keystorepass' not found in config.yml!")) if 'keypass' not in config and config.get('keystore') != 'NONE': nosigningkey = True - logging.critical(_("'keypass' not found in config.py!")) + logging.critical(_("'keypass' not found in config.yml!")) if nosigningkey: raise FDroidException("This command requires a signing key, " + "you can create one using: fdroid update --create-key") @@ -513,7 +556,7 @@ def test_sdk_exists(thisconfig): test_aapt_version(thisconfig['aapt']) return True else: - logging.error(_("'sdk_path' not set in 'config.py'!")) + logging.error(_("'sdk_path' not set in config.yml!")) return False if thisconfig['sdk_path'] == default_config['sdk_path']: logging.error(_('No Android SDK found!')) @@ -3518,19 +3561,24 @@ def load_stats_fdroid_signing_key_fingerprints(): def write_to_config(thisconfig, key, value=None, config_file=None): - '''write a key/value to the local config.py + '''write a key/value to the local config.yml or config.py NOTE: only supports writing string variables. :param thisconfig: config dictionary - :param key: variable name in config.py to be overwritten/added + :param key: variable name in config to be overwritten/added :param value: optional value to be written, instead of fetched from 'thisconfig' dictionary. ''' if value is None: origkey = key + '_orig' value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key] - cfg = config_file if config_file else 'config.py' + if config_file: + cfg = config_file + elif os.path.exists('config.py') and not os.path.exists('config.yml'): + cfg = 'config.py' + else: + cfg = 'config.yml' # load config file, create one if it doesn't exist if not os.path.exists(cfg): @@ -3546,10 +3594,17 @@ def write_to_config(thisconfig, key, value=None, config_file=None): # regex for finding and replacing python string variable # definitions/initializations - pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"') - repl = key + ' = "' + value + '"' - pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'") - repl2 = key + " = '" + value + "'" + if cfg.endswith('.py'): + pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"') + repl = key + ' = "' + value + '"' + pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'") + repl2 = key + " = '" + value + "'" + else: + # assume .yml as default + pattern = re.compile(r'^[\s#]*' + key + r':.*') + repl = yaml.dump({key: value}, default_flow_style=False) + pattern2 = pattern + repl2 = repl # If we replaced this line once, we make sure won't be a # second instance of this line for this key in the document. diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 724487d5..87dfce8b 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -52,7 +52,7 @@ def update_awsbucket(repo_section): subdirectories) to the AWS S3 "bucket". The contents of that subdir of the bucket will first be deleted. - Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey + Requires AWS credentials set in config.yml: awsaccesskeyid, awssecretkey ''' logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "' @@ -160,7 +160,7 @@ def update_awsbucket_libcloud(repo_section): if not config.get('awsaccesskeyid') or not config.get('awssecretkey'): raise FDroidException( - _('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.py!')) + _('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.yml!')) awsbucket = config['awsbucket'] if os.path.exists(USER_S3CFG): diff --git a/fdroidserver/init.py b/fdroidserver/init.py index f53057a0..124555a3 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -36,13 +36,14 @@ options = None def disable_in_config(key, value): - '''write a key/value to the local config.py, then comment it out''' - with open('config.py', 'r') as f: + '''write a key/value to the local config.yml, then comment it out''' + import yaml + with open('config.yml') as f: data = f.read() - pattern = r'\n[\s#]*' + key + r'\s*=\s*"[^"]*"' - repl = '\n#' + key + ' = "' + value + '"' + pattern = r'\n[\s#]*' + key + r':.*' + repl = '\n#' + yaml.dump({key: value}, default_flow_style=False) data = re.sub(pattern, repl, data) - with open('config.py', 'w') as f: + with open('config.yml', 'w') as f: f.writelines(data) @@ -114,13 +115,13 @@ def main(): raise FDroidException(_("Android SDK not found at {path}!") .format(path=test_config['sdk_path'])) - if not os.path.exists('config.py'): + if not os.path.exists('config.yml') and not os.path.exists('config.py'): # 'metadata' and 'tmp' are created in fdroid if not os.path.exists('repo'): os.mkdir('repo') shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir) - shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py') - os.chmod('config.py', 0o0600) + shutil.copyfile(os.path.join(examplesdir, 'config.yml'), 'config.yml') + os.chmod('config.yml', 0o0600) # If android_home is None, test_config['sdk_path'] will be used and # "$ANDROID_HOME" may be used if the env var is set up correctly. # If android_home is not None, the path given from the command line @@ -132,7 +133,7 @@ def main(): logging.info('Try running `fdroid init` in an empty directory.') raise FDroidException('Repository already exists.') - # now that we have a local config.py, read configuration... + # now that we have a local config.yml, read configuration... config = common.read_config(options) # enable apksigner by default so v2/v3 APK signatures validate @@ -143,7 +144,7 @@ def main(): # left for the user to configure # find or generate the keystore for the repo signing key. First try the - # path written in the default config.py. Then check if the user has + # path written in the default config.yml. Then check if the user has # specified a path from the command line, which will trump all others. # Otherwise, create ~/.local/share/fdroidserver and stick it in there. If # keystore is set to NONE, that means that Java will look for keys in a @@ -192,7 +193,7 @@ def main(): f.write('name = OpenSC\nlibrary = ') f.write(opensc_so) f.write('\n') - logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.py.") + logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml.") logging.info("If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update " "--create-key'.") elif os.path.exists(keystore): @@ -202,7 +203,7 @@ def main(): if keydname: to_set.remove('keydname') logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore) - + '\n' + _('Now set these in config.py:') + ' ' + + '\n' + _('Now set these in config.yml:') + ' ' + ', '.join(to_set) + '\n') else: password = common.genpassword() @@ -228,7 +229,7 @@ def main(): msg += '\n Alias for key in store:\t' + repo_keyalias msg += '\n\n' + '''To complete the setup, add your APKs to "%s" then run "fdroid update -c; fdroid update". You might also want to edit -"config.py" to set the URL, repo name, and more. You should also set up +"config.yml" to set the URL, repo name, and more. You should also set up a signing key (a temporary one might have been automatically generated). For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo diff --git a/hooks/pre-commit b/hooks/pre-commit index 94c55888..a197559d 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -8,11 +8,12 @@ exec 1>&2 files=`git diff-index --cached HEAD 2>&1 | sed 's/^:.* //' | uniq | cut -b100-500` if [ -z "$files" ]; then - PY_FILES="fdroid makebuildserver setup.py examples/*.py buildserver/*.py fdroidserver/*.py" + PY_FILES="fdroid makebuildserver setup.py fdroidserver/*.py" PY_TEST_FILES="tests/*.TestCase" SH_FILES="hooks/pre-commit" BASH_FILES="gradlew-fdroid jenkins-build-all jenkins-setup-build-environment jenkins-test completion/bash-completion buildserver/provision-*" RB_FILES="buildserver/Vagrantfile" + YML_FILES="buildserver/*.yml examples/*.yml" else # if actually committing right now, then only run on the files # that are going to be committed at this moment @@ -21,6 +22,7 @@ else SH_FILES= BASH_FILES= RB_FILES= + YML_FILES= for f in $files; do test -e $f || continue @@ -34,6 +36,9 @@ else *.rb) RB_FILES+=" $f" ;; + *.yml) + YML_FILES+=" $f" + ;; *) if head -1 $f | grep '^#!/bin/sh' > /dev/null 2>&1; then SH_FILES+=" $f" @@ -89,6 +94,7 @@ DASH=$(find_command dash) PYFLAKES=$(find_command pyflakes) PEP8=$(find_command pycodestyle pep8) RUBY=$(find_command ruby) +YAMLLINT=$(find_command yamllint) if [ "$PY_FILES $PY_TEST_FILES" != " " ]; then if ! $PYFLAKES $PY_FILES $PY_TEST_FILES; then @@ -129,6 +135,12 @@ for f in $RB_FILES; do fi done +for f in $YML_FILES; do + if ! $YAMLLINT $f 1>/dev/null; then + err ".yml tests failed on $f!" + fi +done + if grep -C 3 'shell=True' fdroidserver/[a-ce-z]*.py; then err "shell=True is too dangerous, there are unfiltered user inputs!" fi diff --git a/jenkins-build-all b/jenkins-build-all index 34b6dfd7..6ae96767 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -81,18 +81,18 @@ else cd fdroiddata fi -echo "build_server_always = True" > config.py -echo "deploy_process_logs = True" >> config.py +echo "build_server_always: true" > config.yml +echo "deploy_process_logs: true" >> config.yml # if the local mediawiki is available, then use it if nc -z -w1 localhost 32445; then wikiflag="--wiki" - echo "wiki_protocol = 'http'" >> config.py - echo "wiki_server = 'localhost:32445'" >> config.py - echo "wiki_path = '/mediawiki/'" >> config.py - echo "wiki_user = 'fdroid'" >> config.py - echo "wiki_password = 'update.TestCase'" >> config.py + echo "wiki_protocol: http" >> config.yml + echo "wiki_server: localhost:32445" >> config.yml + echo "wiki_path: /mediawiki/" >> config.yml + echo "wiki_user: fdroid" >> config.yml + echo "wiki_password: update.TestCase" >> config.yml else - sed -i '/^wiki_/d' config.py + sed -i '/^wiki_/d' config.yml fi $WORKSPACE/fdroid build --verbose --latest --no-tarball --all $wikiflag diff --git a/jenkins-setup-build-environment b/jenkins-setup-build-environment index c47359b0..489717b2 100755 --- a/jenkins-setup-build-environment +++ b/jenkins-setup-build-environment @@ -98,7 +98,7 @@ else fi cd fdroiddata -echo "build_server_always = True" > config.py +echo "build_server_always: true" > config.yml if [ -z $ANDROID_HOME ]; then if [ -e ~/.android/bashrc ]; then diff --git a/jenkins-test b/jenkins-test index 2790a1fe..4ea8702b 100755 --- a/jenkins-test +++ b/jenkins-test @@ -44,7 +44,7 @@ fi # this is set up and managed by jenkins-build-all cd $WORKSPACE/fdroiddata -rm -f config.py keystore.jks keystore.p12 +rm -f config.py config.yml keystore.jks keystore.p12 ../fdroid init --verbose export GNUPGHOME=$WORKSPACE/tests/gnupghome @@ -54,13 +54,13 @@ if [ ! -e $GNUPGHOME/private-keys-v1.d ]; then fi gpg --import $GNUPGHOME/secring.gpg -echo "build_server_always = True" >> config.py -echo "deploy_process_logs = True" >> config.py -echo "make_current_version_link = False" >> config.py -echo "gpghome = '$GNUPGHOME'" >> config.py -echo "gpgkey = 'CE71F7FB'" >> config.py -chmod 0600 config.py -sed -i '/\s*repo_key_sha256\s*=.*/d' config.py +echo "build_server_always: true" >> config.yml +echo "deploy_process_logs: true" >> config.yml +echo "make_current_version_link: false" >> config.yml +echo "gpghome: $GNUPGHOME" >> config.yml +echo "gpgkey: CE71F7FB" >> config.yml +chmod 0600 config.yml +sed -i '/\s*repo_key_sha256:.*/d' config.yml # publish process when building and signing are on separate machines test -d repo || mkdir repo @@ -71,9 +71,9 @@ test -d archive || mkdir archive ../fdroid gpgsign # when everything is copied over to run on BUILD machine, # which does not have a keyring, only a cached pubkey -echo "repo_pubkey = '308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a'" >> config.py +echo "repo_pubkey: 308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" >> config.yml ../fdroid update --nosign -sed -i '/^repo_pubkey = /d' config.py +sed -i '/^repo_pubkey: /d' config.yml # when everything is copied over to run on SIGN machine ../fdroid signindex --verbose diff --git a/setup.py b/setup.py index 363f686a..26722299 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def get_data_files(): data = fp.read() data_files.append((data_prefix + '/share/doc/fdroidserver/examples', - ['buildserver/config.buildserver.py', ] + ['buildserver/config.buildserver.yml', ] + re.findall(r'include (examples/.*)', data))) for f in re.findall(r'include (locale/[a-z][a-z][a-zA-Z_]*/LC_MESSAGES/fdroidserver.mo)', data): diff --git a/tests/common.TestCase b/tests/common.TestCase index ff7befb1..eff088d6 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -47,6 +47,7 @@ class CommonTest(unittest.TestCase): if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) os.chdir(self.basedir) + fdroidserver.common.config = None def test_parse_human_readable_size(self): for k, v in ((9827, 9827), (123.456, 123), ('123b', 123), ('1.2', 1), @@ -1409,6 +1410,132 @@ class CommonTest(unittest.TestCase): self.assertIsNotNone(result) self.assertNotEqual(result, '') + def test_with_no_config(self): + """It should set defaults if no config file is found""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + self.assertFalse(os.path.exists('config.yml')) + self.assertFalse(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual(None, config.get('apksigner')) + self.assertIsNotNone(config.get('char_limits')) + + def test_with_config_yml(self): + """Make sure it is possible to use config.yml alone.""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.yml', 'w') as fp: + fp.write('apksigner: yml') + self.assertTrue(os.path.exists('config.yml')) + self.assertFalse(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual('yml', config.get('apksigner')) + + def test_with_config_yml_with_env_var(self): + """Make sure it is possible to use config.yml alone.""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + os.environ['SECRET'] = 'mysecretpassword' + with open('config.yml', 'w') as fp: + fp.write("""keypass: {'env': 'SECRET'}""") + self.assertTrue(os.path.exists('config.yml')) + self.assertFalse(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual(os.getenv('SECRET', 'fail'), config.get('keypass')) + + def test_with_config_py(self): + """Make sure it is still possible to use config.py alone.""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.py', 'w') as fp: + fp.write('apksigner = "py"') + self.assertFalse(os.path.exists('config.yml')) + self.assertTrue(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual("py", config.get('apksigner')) + + def test_config_perm_warning(self): + """Exercise the code path that issues a warning about unsafe permissions.""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.yml', 'w') as fp: + fp.write('keystore: foo.jks') + self.assertTrue(os.path.exists(fp.name)) + os.chmod(fp.name, 0o666) + fdroidserver.common.read_config(fdroidserver.common.options) + os.remove(fp.name) + fdroidserver.common.config = None + + with open('config.py', 'w') as fp: + fp.write('keystore = "foo.jks"') + self.assertTrue(os.path.exists(fp.name)) + os.chmod(fp.name, 0o666) + fdroidserver.common.read_config(fdroidserver.common.options) + + def test_with_both_config_yml_py(self): + """If config.yml and config.py are present, config.py should be ignored.""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.yml', 'w') as fp: + fp.write('apksigner: yml') + with open('config.py', 'w') as fp: + fp.write('apksigner = "py"') + self.assertTrue(os.path.exists('config.yml')) + self.assertTrue(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual('yml', config.get('apksigner')) + + def test_write_to_config_yml(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.yml', 'w') as fp: + fp.write('apksigner: yml') + self.assertTrue(os.path.exists(fp.name)) + self.assertFalse(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertFalse('keypass' in config) + self.assertEqual('yml', config.get('apksigner')) + fdroidserver.common.write_to_config(config, 'keypass', 'mysecretpassword') + with open(fp.name) as fp: + print(fp.read()) + fdroidserver.common.config = None + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual('mysecretpassword', config['keypass']) + + def test_write_to_config_py(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.py', 'w') as fp: + fp.write('apksigner = "py"') + self.assertTrue(os.path.exists(fp.name)) + self.assertFalse(os.path.exists('config.yml')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertFalse('keypass' in config) + self.assertEqual('py', config.get('apksigner')) + fdroidserver.common.write_to_config(config, 'keypass', 'mysecretpassword') + fdroidserver.common.config = None + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual('mysecretpassword', config['keypass']) + + def test_config_dict_with_int_keys(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.yml', 'w') as fp: + fp.write('java_paths:\n 8: /usr/lib/jvm/java-8-openjdk\n') + self.assertTrue(os.path.exists(fp.name)) + self.assertFalse(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual('/usr/lib/jvm/java-8-openjdk', config.get('java_paths', {}).get('8')) + + def test_loading_config_buildserver_yml(self): + """Smoke check to make sure this file is properly parsed""" + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + shutil.copy(os.path.join(self.basedir, '..', 'buildserver', 'config.buildserver.yml'), + 'config.yml') + self.assertFalse(os.path.exists('config.py')) + fdroidserver.common.read_config(fdroidserver.common.options) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/init.TestCase b/tests/init.TestCase new file mode 100755 index 00000000..f080ff74 --- /dev/null +++ b/tests/init.TestCase @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# http://www.drdobbs.com/testing/unit-testing-with-python/240165163 + +import inspect +import logging +import os +import optparse +import sys +import tempfile +import unittest + + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.init + + +class InitTest(unittest.TestCase): + '''fdroidserver/init.py''' + + def setUp(self): + logging.basicConfig(level=logging.DEBUG) + self.basedir = os.path.join(localmodule, 'tests') + self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + os.chdir(self.basedir) + fdroidserver.init.config = None + + def test_disable_in_config(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + with open('config.yml', 'w') as fp: + fp.write('keystore: NONE\n') + fp.write('keypass: mysupersecrets\n') + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual('NONE', config['keystore']) + self.assertEqual('mysupersecrets', config['keypass']) + fdroidserver.init.disable_in_config('keypass', 'comment') + with open(fp.name) as fp: + self.assertTrue('#keypass:' in fp.read()) + fdroidserver.common.config = None + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertIsNone(config.get('keypass')) + + +if __name__ == "__main__": + os.chdir(os.path.dirname(__file__)) + + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.init.options, args) = parser.parse_args(['--verbose']) + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(InitTest)) + unittest.main(failfast=False) diff --git a/tests/run-tests b/tests/run-tests index 282b8e70..8645a366 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -61,8 +61,8 @@ fdroid_init_with_prebuilt_keystore() { keystore="$1" fi $fdroid init --keystore $keystore --repo-keyalias=sova - echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py - echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py + echo 'keystorepass: r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' >> config.yml + echo 'keypass: r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' >> config.yml } # the < is reverse since 0 means success in exit codes @@ -170,10 +170,10 @@ if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then cd $REPOROOT cp $WORKSPACE/tests/keystore.jks $REPOROOT/ $fdroid init --keystore keystore.jks --repo-keyalias=sova - echo 'make_current_version_link = True' >> config.py - echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py - echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py - echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py + echo 'make_current_version_link: true' >> config.yml + echo 'keystorepass: r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' >> config.yml + echo 'keypass: r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=' >> config.yml + echo 'keydname: "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.yml test -d archive || mkdir archive test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ @@ -199,8 +199,8 @@ REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore -$sed -i.tmp 's,^ *repo_description.*,repo_description = """获取已安装在您的设备上的应用的,' config.py -echo "mirrors = ('https://foo.bar/fdroid', 'http://secret.onion/fdroid')" >> config.py +$sed -i.tmp 's,^ *repo_description.*,repo_description: |\n 获取已安装在您的设备上的应用的,' config.yml +echo "mirrors: ['https://foo.bar/fdroid', 'http://secret.onion/fdroid']" >> config.yml mkdir metadata cp $WORKSPACE/tests/urzip.apk $WORKSPACE/tests/bad-unicode*.apk repo/ cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ @@ -258,11 +258,11 @@ fdroid_init_with_prebuilt_keystore cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/ cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME chmod 0700 $GNUPGHOME -echo "install_list = 'org.adaway'" >> config.py -echo "uninstall_list = ('com.android.vending', 'com.facebook.orca',)" >> config.py -echo "gpghome = '$GNUPGHOME'" >> config.py -echo "gpgkey = 'CE71F7FB'" >> config.py -echo "mirrors = ('http://foobarfoobarfoobar.onion/fdroid','https://foo.bar/fdroid',)" >> config.py +echo "install_list: org.adaway" >> config.yml +echo "uninstall_list: [com.android.vending, com.facebook.orca]" >> config.yml +echo "gpghome: $GNUPGHOME" >> config.yml +echo "gpgkey: CE71F7FB" >> config.yml +echo "mirrors: ['http://foobarfoobarfoobar.onion/fdroid', 'https://foo.bar/fdroid']" >> config.yml $fdroid update --verbose --pretty test -e repo/index.xml test -e repo/index.jar @@ -298,7 +298,7 @@ echo_header 'test moving lots of APKs to the archive' REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore -$sed -i.tmp '/allow_disabled_algorithms/d' config.py +$sed -i.tmp '/allow_disabled_algorithms/d' config.yml test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/*.yml metadata/ echo 'Summary: good test version of urzip' > metadata/info.guardianproject.urzip.yml @@ -310,7 +310,7 @@ cp $WORKSPACE/tests/urzip.apk \ $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \ $WORKSPACE/tests/repo/obb.main.twoversions_110161[357].apk \ repo/ -$sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py +$sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml $fdroid update --pretty --nosign if which apksigner; then @@ -333,7 +333,7 @@ if ! which apksigner; then cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ test -d repo || mkdir repo cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/ - $sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py + $sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 0 @@ -419,7 +419,7 @@ cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ $sed -i.tmp '/ArchivePolicy:/d' metadata/com.politedroid.yml test -d repo || mkdir repo cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/ -$sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py +$sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 1 @@ -433,7 +433,7 @@ test -e repo/com.politedroid_4.apk test -e repo/com.politedroid_5.apk test -e repo/com.politedroid_6.apk -$sed -i.tmp 's,archive_older = 3,archive_older = 1,' config.py +$sed -i.tmp 's,archive_older: 3,archive_older: 1,' config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 3 test `grep '' repo/index.xml | wc -l` -eq 1 @@ -494,8 +494,8 @@ echo_header 'test allowing disabled signatures in repo and archive' REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore -echo 'allow_disabled_algorithms = True' >> config.py -$sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py +echo 'allow_disabled_algorithms: true' >> config.yml +$sed -i.tmp 's,archive_older: [0-9],archive_older: 3,' config.yml test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ echo 'Summary: good test version of urzip' > metadata/info.guardianproject.urzip.yml @@ -531,7 +531,7 @@ test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk test -e archive/urzip-badsig.apk if ! which apksigner; then - $sed -i.tmp '/allow_disabled_algorithms/d' config.py + $sed -i.tmp '/allow_disabled_algorithms/d' config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 5 test `grep '' repo/index.xml | wc -l` -eq 3 @@ -557,7 +557,7 @@ if ! which apksigner; then fi # test unarchiving when disabled_algorithms are allowed again -echo 'allow_disabled_algorithms = True' >> config.py +echo 'allow_disabled_algorithms: true' >> config.yml $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 2 test `grep '' repo/index.xml | wc -l` -eq 6 @@ -587,7 +587,7 @@ echo_header 'rename apks with `fdroid update --rename-apks`, --nosign for speed' REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore -echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py +echo 'keydname: "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.yml test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ test -d repo || mkdir repo @@ -684,11 +684,11 @@ echo "Description: |" >> metadata/fake.yml echo " this is fake" >> metadata/fake.yml # fake that no JDKs are available -echo 'java_paths = {}' > config.py +echo 'java_paths: {}' > config.yml LOCAL_COPY_DIR=`create_test_dir`/fdroid mkdir -p $LOCAL_COPY_DIR/repo -echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py +echo "local_copy_dir: $LOCAL_COPY_DIR" >> config.yml $fdroid checkupdates --allow-dirty which gpg && $fdroid gpgsign @@ -775,7 +775,7 @@ $fdroid deploy --local-copy-dir=$LOCALCOPYDIR NEWREPOROOT=`create_test_dir` cd $NEWREPOROOT fdroid_init_with_prebuilt_keystore -echo "sync_from_local_copy_dir = True" >> config.py +echo "sync_from_local_copy_dir: true" >> config.yml $fdroid deploy --local-copy-dir=$LOCALCOPYDIR @@ -844,7 +844,7 @@ KEYSTORE=$REPOROOT/keystore.p12 cd $REPOROOT $fdroid init --keystore $KEYSTORE --android-home $FAKE_ANDROID_HOME set +e -grep $FAKE_ANDROID_HOME $REPOROOT/config.py +grep $FAKE_ANDROID_HOME $REPOROOT/config.yml if [ $? -ne 0 ]; then echo "the value set in --android-home '$FAKE_ANDROID_HOME' should override ANDROID_HOME '$ANDROID_HOME'" exit 1 @@ -1023,10 +1023,10 @@ set -e # now set up fake, non-working keystore setup touch $KEYSTORE -echo "keystore = \"$KEYSTORE\"" >> config.py -echo 'repo_keyalias = "foo"' >> config.py -echo 'keystorepass = "foo"' >> config.py -echo 'keypass = "foo"' >> config.py +echo "keystore: $KEYSTORE" >> config.yml +echo 'repo_keyalias: foo' >> config.yml +echo 'keystorepass: foo' >> config.yml +echo 'keypass: foo' >> config.yml set +e $fdroid update --create-metadata --verbose if [ $? -eq 0 ]; then @@ -1047,7 +1047,7 @@ GNUPGHOME=$REPOROOT/gnupghome cd $REPOROOT fdroid_init_with_prebuilt_keystore cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/ -echo "binary_transparency_remote = '$GIT_REMOTE'" >> config.py +echo "binary_transparency_remote: $GIT_REMOTE" >> config.yml $fdroid update --verbose if have_git_2_3; then $fdroid deploy --verbose @@ -1084,7 +1084,7 @@ test -e tmp/apkcache.json grep -F '> config.py +echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/ $fdroid update --create-metadata @@ -1155,7 +1155,7 @@ test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_6.apk before=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'` -echo "git_mirror_size_limit = '60kb'" >> config.py +echo "git_mirror_size_limit: 60kb" >> config.yml $fdroid update $fdroid deploy test -e $REPOROOT/archive/com.politedroid_3.apk @@ -1197,9 +1197,9 @@ if have_git_2_3; then fdroid_init_with_prebuilt_keystore cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $OFFLINE_ROOT/ - echo "mirrors = ['http://foo.bar/fdroid', 'http://asdflkdsfjafdsdfhkjh.onion/fdroid']" >> config.py - echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py - echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py + echo "mirrors: ['http://foo.bar/fdroid', 'http://asdflkdsfjafdsdfhkjh.onion/fdroid']" >> config.yml + echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml + echo "local_copy_dir: $LOCAL_COPY_DIR" >> config.yml $fdroid update --pretty grep -F '' repo/index.xml @@ -1212,12 +1212,12 @@ if have_git_2_3; then $fdroid deploy --verbose grep -F '> config.py - echo "sync_from_local_copy_dir = True" >> config.py - echo "serverwebroots = '$SERVERWEBROOT'" >> config.py - echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py - echo "local_copy_dir = '$LOCAL_COPY_DIR'" >> config.py - echo "binary_transparency_remote = '$BINARY_TRANSPARENCY_REMOTE'" >> config.py + echo "local_copy_dir: $LOCAL_COPY_DIR" >> config.yml + echo "sync_from_local_copy_dir: True" >> config.yml + echo "serverwebroots: $SERVERWEBROOT" >> config.yml + echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml + echo "local_copy_dir: $LOCAL_COPY_DIR" >> config.yml + echo "binary_transparency_remote: $BINARY_TRANSPARENCY_REMOTE" >> config.yml $fdroid deploy --verbose cd $BINARY_TRANSPARENCY_REMOTE [ `git rev-list --count HEAD` == "1" ] @@ -1232,7 +1232,7 @@ echo_header 'test extracting and publishing with developer signature' REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore -echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py +echo 'keydname: "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.yml test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata/ test -d repo || mkdir repo