diff --git a/fdroidserver/__init__.py b/fdroidserver/__init__.py index 1e1cd42f..72a9c38d 100644 --- a/fdroidserver/__init__.py +++ b/fdroidserver/__init__.py @@ -1,4 +1,3 @@ - import gettext import glob import os @@ -8,7 +7,9 @@ import sys # support running straight from git and standard installs rootpaths = [ os.path.realpath(os.path.join(os.path.dirname(__file__), '..')), - os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share')), + os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share') + ), os.path.join(sys.prefix, 'share'), ] diff --git a/fdroidserver/gpgsign.py b/fdroidserver/gpgsign.py index d5d92c6d..447c3906 100644 --- a/fdroidserver/gpgsign.py +++ b/fdroidserver/gpgsign.py @@ -59,7 +59,9 @@ def main(): signed = [] for output_dir in repodirs: if not os.path.isdir(output_dir): - raise FDroidException(_("Missing output directory") + " '" + output_dir + "'") + raise FDroidException( + _("Missing output directory") + " '" + output_dir + "'" + ) # Process any apks that are waiting to be signed... for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))): diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 6f6f8c38..c429f096 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -38,6 +38,7 @@ options = None def disable_in_config(key, value): """Write a key/value to the local config.yml, then comment it out.""" import yaml + with open('config.yml') as f: data = f.read() pattern = r'\n[\s#]*' + key + r':.*' @@ -54,16 +55,33 @@ def main(): # Parse command line... parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument("-d", "--distinguished-name", default=None, - help=_("X.509 'Distinguished Name' used when generating keys")) - parser.add_argument("--keystore", default=None, - help=_("Path to the keystore for the repo signing key")) - parser.add_argument("--repo-keyalias", default=None, - help=_("Alias of the repo signing key in the keystore")) - parser.add_argument("--android-home", default=None, - help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)")) - parser.add_argument("--no-prompt", action="store_true", default=False, - help=_("Do not prompt for Android SDK path, just fail")) + parser.add_argument( + "-d", + "--distinguished-name", + default=None, + help=_("X.509 'Distinguished Name' used when generating keys"), + ) + parser.add_argument( + "--keystore", + default=None, + help=_("Path to the keystore for the repo signing key"), + ) + parser.add_argument( + "--repo-keyalias", + default=None, + help=_("Alias of the repo signing key in the keystore"), + ) + parser.add_argument( + "--android-home", + default=None, + help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)"), + ) + parser.add_argument( + "--no-prompt", + action="store_true", + default=False, + help=_("Do not prompt for Android SDK path, just fail"), + ) options = parser.parse_args() fdroiddir = os.getcwd() @@ -81,8 +99,9 @@ def main(): # and if the user leaves it blank, ignore and move on. default_sdk_path = '' if sys.platform == 'win32' or sys.platform == 'cygwin': - p = os.path.join(os.getenv('USERPROFILE'), - 'AppData', 'Local', 'Android', 'android-sdk') + p = os.path.join( + os.getenv('USERPROFILE'), 'AppData', 'Local', 'Android', 'android-sdk' + ) elif sys.platform == 'darwin': # on OSX, Homebrew is common and has an easy path to detect p = '/usr/local/opt/android-sdk' @@ -96,10 +115,13 @@ def main(): test_config['sdk_path'] = default_sdk_path if not common.test_sdk_exists(test_config): - del(test_config['sdk_path']) + del (test_config['sdk_path']) while not options.no_prompt: try: - s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path) + s = input( + _('Enter the path to the Android SDK (%s) here:\n> ') + % default_sdk_path + ) except KeyboardInterrupt: print('') sys.exit(1) @@ -112,8 +134,9 @@ def main(): default_sdk_path = '' if test_config.get('sdk_path') and not common.test_sdk_exists(test_config): - raise FDroidException(_("Android SDK not found at {path}!") - .format(path=test_config['sdk_path'])) + raise FDroidException( + _("Android SDK not found at {path}!").format(path=test_config['sdk_path']) + ) if not os.path.exists('config.yml') and not os.path.exists('config.py'): # 'metadata' and 'tmp' are created in fdroid @@ -124,12 +147,14 @@ def main(): shutil.copyfile(example_config_yml, 'config.yml') else: from pkg_resources import get_distribution + versionstr = get_distribution('fdroidserver').version if not versionstr: versionstr = 'master' with open('config.yml', 'w') as fp: - fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/' - + versionstr + '/examples/config.yml\n') + fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/') + fp.write(versionstr) + fp.write('/examples/config.yml\n') os.chmod('config.yml', 0o0600) # If android_home is None, test_config['sdk_path'] will be used and # "$ANDROID_HOME" may be used if the env var is set up correctly. @@ -138,7 +163,9 @@ def main(): if 'sdk_path' in test_config: common.write_to_config(test_config, 'sdk_path', options.android_home) else: - logging.warning('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') + logging.warning( + 'Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...' + ) logging.info('Try running `fdroid init` in an empty directory.') raise FDroidException('Repository already exists.') @@ -162,8 +189,9 @@ def main(): else: keystore = os.path.abspath(options.keystore) if not os.path.exists(keystore): - logging.info('"' + keystore - + '" does not exist, creating a new keystore there.') + logging.info( + '"' + keystore + '" does not exist, creating a new keystore there.' + ) common.write_to_config(test_config, 'keystore', keystore) repo_keyalias = None keydname = None @@ -174,12 +202,19 @@ def main(): keydname = options.distinguished_name common.write_to_config(test_config, 'keydname', keydname) if keystore == 'NONE': # we're using a smartcard - common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default + common.write_to_config( + test_config, 'repo_keyalias', '1' + ) # seems to be the default disable_in_config('keypass', 'never used with smartcard') - common.write_to_config(test_config, 'smartcardoptions', - ('-storetype PKCS11 ' - + '-providerClass sun.security.pkcs11.SunPKCS11 ' - + '-providerArg opensc-fdroid.cfg')) + common.write_to_config( + test_config, + 'smartcardoptions', + ( + '-storetype PKCS11 ' + + '-providerClass sun.security.pkcs11.SunPKCS11 ' + + '-providerArg opensc-fdroid.cfg' + ), + ) # find opensc-pkcs11.so if not os.path.exists('opensc-fdroid.cfg'): if os.path.exists('/usr/lib/opensc-pkcs11.so'): @@ -187,29 +222,43 @@ def main(): elif os.path.exists('/usr/lib64/opensc-pkcs11.so'): opensc_so = '/usr/lib64/opensc-pkcs11.so' else: - files = glob.glob('/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so') + files = glob.glob( + '/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so' + ) if len(files) > 0: opensc_so = files[0] else: opensc_so = '/usr/lib/opensc-pkcs11.so' - logging.warning('No OpenSC PKCS#11 module found, ' - + 'install OpenSC then edit "opensc-fdroid.cfg"!') + logging.warning( + 'No OpenSC PKCS#11 module found, ' + + 'install OpenSC then edit "opensc-fdroid.cfg"!' + ) with open('opensc-fdroid.cfg', 'w') as f: f.write('name = OpenSC\nlibrary = ') f.write(opensc_so) f.write('\n') - logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml.") - logging.info("If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update " - "--create-key'.") + logging.info( + "Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml." + ) + logging.info( + "If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update " + "--create-key'." + ) elif os.path.exists(keystore): to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname'] if repo_keyalias: to_set.remove('repo_keyalias') if keydname: to_set.remove('keydname') - logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore) - + '\n' + _('Now set these in config.yml:') + ' ' - + ', '.join(to_set) + '\n') + logging.warning( + '\n' + + _('Using existing keystore "{path}"').format(path=keystore) + + '\n' + + _('Now set these in config.yml:') + + ' ' + + ', '.join(to_set) + + '\n' + ) else: password = common.genpassword() c = dict(test_config) @@ -229,11 +278,17 @@ def main(): msg += '\n ' + _('Keystore for signing key:\t') + keystore if repo_keyalias is not None: msg += '\n Alias for key in store:\t' + repo_keyalias - msg += '\n\n' + '''To complete the setup, add your APKs to "%s" + msg += '\n\n' + msg += ( + _( + '''To complete the setup, add your APKs to "%s" then run "fdroid update -c; fdroid update". You might also want to edit "config.yml" to set the URL, repo name, and more. You should also set up a signing key (a temporary one might have been automatically generated). For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo -and https://f-droid.org/docs/Signing_Process''' % os.path.join(fdroiddir, 'repo') +and https://f-droid.org/docs/Signing_Process''' + ) + % os.path.join(fdroiddir, 'repo') + ) logging.info(msg) diff --git a/fdroidserver/mirror.py b/fdroidserver/mirror.py index c34bd9a2..3187a3d3 100644 --- a/fdroidserver/mirror.py +++ b/fdroidserver/mirror.py @@ -33,8 +33,15 @@ def _run_wget(path, urls): with open(urls_file, 'w') as fp: for url in urls: fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename - subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"', - '--input-file=' + urls_file]) + subprocess.call( + [ + 'wget', + verbose, + '--continue', + '--user-agent="fdroid mirror"', + '--input-file=' + urls_file, + ] + ) os.remove(urls_file) @@ -43,21 +50,47 @@ def main(): parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument("url", nargs='?', - help=_('Base URL to mirror, can include the index signing key ' - + 'using the query string: ?fingerprint=')) - parser.add_argument("--all", action='store_true', default=False, - help=_("Mirror the full repo and archive, all file types.")) - parser.add_argument("--archive", action='store_true', default=False, - help=_("Also mirror the full archive section")) - parser.add_argument("--build-logs", action='store_true', default=False, - help=_("Include the build logs in the mirror")) - parser.add_argument("--pgp-signatures", action='store_true', default=False, - help=_("Include the PGP signature .asc files in the mirror")) - parser.add_argument("--src-tarballs", action='store_true', default=False, - help=_("Include the source tarballs in the mirror")) - parser.add_argument("--output-dir", default=None, - help=_("The directory to write the mirror to")) + parser.add_argument( + "url", + nargs='?', + help=_( + 'Base URL to mirror, can include the index signing key ' + + 'using the query string: ?fingerprint=' + ), + ) + parser.add_argument( + "--all", + action='store_true', + default=False, + help=_("Mirror the full repo and archive, all file types."), + ) + parser.add_argument( + "--archive", + action='store_true', + default=False, + help=_("Also mirror the full archive section"), + ) + parser.add_argument( + "--build-logs", + action='store_true', + default=False, + help=_("Include the build logs in the mirror"), + ) + parser.add_argument( + "--pgp-signatures", + action='store_true', + default=False, + help=_("Include the PGP signature .asc files in the mirror"), + ) + parser.add_argument( + "--src-tarballs", + action='store_true', + default=False, + help=_("Include the source tarballs in the mirror"), + ) + parser.add_argument( + "--output-dir", default=None, help=_("The directory to write the mirror to") + ) options = parser.parse_args() if options.all: @@ -77,24 +110,31 @@ def main(): def _append_to_url_path(*args): """Append the list of path components to URL, keeping the rest the same.""" newpath = posixpath.join(path, *args) - return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment)) + return urllib.parse.urlunparse( + (scheme, hostname, newpath, params, query, fragment) + ) if fingerprint: config = common.read_config(options) if not ('jarsigner' in config or 'apksigner' in config): - logging.error(_('Java JDK not found! Install in standard location or set java_paths!')) + logging.error( + _('Java JDK not found! Install in standard location or set java_paths!') + ) sys.exit(1) def _get_index(section, etag=None): url = _append_to_url_path(section) data, etag = index.download_repo_index(url, etag=etag) return data, etag, _append_to_url_path(section, 'index-v1.jar') + else: + def _get_index(section, etag=None): import io import json import zipfile from . import net + url = _append_to_url_path(section, 'index-v1.jar') content, etag = net.http_get(url) with zipfile.ZipFile(io.BytesIO(content)) as zip: @@ -107,21 +147,30 @@ def main(): ip = ipaddress.ip_address(hostname) except ValueError: pass - if hostname == 'f-droid.org' \ - or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]): - print(_('ERROR: this command should never be used to mirror f-droid.org!\n' - 'A full mirror of f-droid.org requires more than 200GB.')) + if hostname == 'f-droid.org' or ( + ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2] + ): + print( + _( + 'ERROR: this command should never be used to mirror f-droid.org!\n' + 'A full mirror of f-droid.org requires more than 200GB.' + ) + ) sys.exit(1) path = path.rstrip('/') if path.endswith('repo') or path.endswith('archive'): - logging.warning(_('Do not include "{path}" in URL!') - .format(path=path.split('/')[-1])) + logging.warning( + _('Do not include "{path}" in URL!').format(path=path.split('/')[-1]) + ) elif not path.endswith('fdroid'): - logging.warning(_('{url} does not end with "fdroid", check the URL path!') - .format(url=options.url)) + logging.warning( + _('{url} does not end with "fdroid", check the URL path!').format( + url=options.url + ) + ) - icondirs = ['icons', ] + icondirs = ['icons'] for density in update.screen_densities: icondirs.append('icons-' + density) @@ -134,7 +183,7 @@ def main(): if options.archive: sections = ('repo', 'archive') else: - sections = ('repo', ) + sections = ('repo',) for section in sections: sectiondir = os.path.join(basedir, section) @@ -152,23 +201,29 @@ def main(): for packageName, packageList in data['packages'].items(): for package in packageList: to_fetch = [] - keys = ['apkName', ] + keys = ['apkName'] if options.src_tarballs: keys.append('srcname') for k in keys: if k in package: to_fetch.append(package[k]) elif k == 'apkName': - logging.error(_('{appid} is missing {name}') - .format(appid=package['packageName'], name=k)) + logging.error( + _('{appid} is missing {name}').format( + appid=package['packageName'], name=k + ) + ) for f in to_fetch: - if not os.path.exists(f) \ - or (f.endswith('.apk') and os.path.getsize(f) != package['size']): + if not os.path.exists(f) or ( + f.endswith('.apk') and os.path.getsize(f) != package['size'] + ): urls.append(_append_to_url_path(section, f)) if options.pgp_signatures: urls.append(_append_to_url_path(section, f + '.asc')) if options.build_logs and f.endswith('.apk'): - urls.append(_append_to_url_path(section, f[:-4] + '.log.gz')) + urls.append( + _append_to_url_path(section, f[:-4] + '.log.gz') + ) _run_wget(sectiondir, urls) @@ -181,7 +236,7 @@ def main(): for k in update.GRAPHIC_NAMES: f = d.get(k) if f: - filepath_tuple = components + (f, ) + filepath_tuple = components + (f,) urls.append(_append_to_url_path(*filepath_tuple)) _run_wget(os.path.join(basedir, *components), urls) for k in update.SCREENSHOT_DIRS: @@ -190,14 +245,16 @@ def main(): if filelist: components = (section, app['packageName'], locale, k) for f in filelist: - filepath_tuple = components + (f, ) + filepath_tuple = components + (f,) urls.append(_append_to_url_path(*filepath_tuple)) _run_wget(os.path.join(basedir, *components), urls) urls = dict() for app in data['apps']: if 'icon' not in app: - logging.error(_('no "icon" in {appid}').format(appid=app['packageName'])) + logging.error( + _('no "icon" in {appid}').format(appid=app['packageName']) + ) continue icon = app['icon'] for icondir in icondirs: diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 50239e20..85ef58fc 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -54,27 +54,72 @@ def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE): p12 = os.path.join(tmp_dir, '.keystore.p12') _config = dict() common.fill_config_defaults(_config) - subprocess.check_call([_config['keytool'], '-importkeystore', - '-srckeystore', keystore, '-srcalias', KEY_ALIAS, - '-srcstorepass', PASSWORD, '-srckeypass', PASSWORD, - '-destkeystore', p12, '-destalias', KEY_ALIAS, - '-deststorepass', PASSWORD, '-destkeypass', PASSWORD, - '-deststoretype', 'PKCS12'], - env={'LC_ALL': 'C.UTF-8'}) - subprocess.check_call(['openssl', 'pkcs12', '-in', p12, '-out', key_pem, - '-passin', 'pass:' + PASSWORD, '-passout', 'pass:' + PASSWORD], - env={'LC_ALL': 'C.UTF-8'}) - subprocess.check_call(['openssl', 'rsa', '-in', key_pem, '-out', privkey, - '-passin', 'pass:' + PASSWORD], - env={'LC_ALL': 'C.UTF-8'}) + subprocess.check_call( + [ + _config['keytool'], + '-importkeystore', + '-srckeystore', + keystore, + '-srcalias', + KEY_ALIAS, + '-srcstorepass', + PASSWORD, + '-srckeypass', + PASSWORD, + '-destkeystore', + p12, + '-destalias', + KEY_ALIAS, + '-deststorepass', + PASSWORD, + '-destkeypass', + PASSWORD, + '-deststoretype', + 'PKCS12', + ], + env={'LC_ALL': 'C.UTF-8'}, + ) + subprocess.check_call( + [ + 'openssl', + 'pkcs12', + '-in', + p12, + '-out', + key_pem, + '-passin', + 'pass:' + PASSWORD, + '-passout', + 'pass:' + PASSWORD, + ], + env={'LC_ALL': 'C.UTF-8'}, + ) + subprocess.check_call( + [ + 'openssl', + 'rsa', + '-in', + key_pem, + '-out', + privkey, + '-passin', + 'pass:' + PASSWORD, + ], + env={'LC_ALL': 'C.UTF-8'}, + ) os.remove(key_pem) os.remove(p12) os.chmod(privkey, 0o600) # os.umask() should cover this, but just in case rsakey = paramiko.RSAKey.from_private_key_file(privkey) - fingerprint = base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()).decode('ascii').rstrip('=') - ssh_private_key_file = os.path.join(tmp_dir, 'debug_keystore_' - + fingerprint.replace('/', '_') + '_id_rsa') + fingerprint = ( + base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()) + .decode('ascii') + .rstrip('=') + ) + ssh_private_key_file = os.path.join( + tmp_dir, 'debug_keystore_' + fingerprint.replace('/', '_') + '_id_rsa' + ) shutil.move(privkey, ssh_private_key_file) pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file @@ -90,20 +135,46 @@ def main(): parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument("--keystore", default=KEYSTORE_FILE, - help=_("Specify which debug keystore file to use.")) - parser.add_argument("--show-secret-var", action="store_true", default=False, - help=_("Print the secret variable to the terminal for easy copy/paste")) - parser.add_argument("--keep-private-keys", action="store_true", default=False, - help=_("Do not remove the private keys generated from the keystore")) - parser.add_argument("--no-deploy", action="store_true", default=False, - help=_("Do not deploy the new files to the repo")) - parser.add_argument("--file", default='app/build/outputs/apk/*.apk', - help=_('The file to be included in the repo (path or glob)')) - parser.add_argument("--no-checksum", action="store_true", default=False, - help=_("Don't use rsync checksums")) - parser.add_argument("--archive-older", type=int, default=20, - help=_("Set maximum releases in repo before older ones are archived")) + parser.add_argument( + "--keystore", + default=KEYSTORE_FILE, + help=_("Specify which debug keystore file to use."), + ) + parser.add_argument( + "--show-secret-var", + action="store_true", + default=False, + help=_("Print the secret variable to the terminal for easy copy/paste"), + ) + parser.add_argument( + "--keep-private-keys", + action="store_true", + default=False, + help=_("Do not remove the private keys generated from the keystore"), + ) + parser.add_argument( + "--no-deploy", + action="store_true", + default=False, + help=_("Do not deploy the new files to the repo"), + ) + parser.add_argument( + "--file", + default='app/build/outputs/apk/*.apk', + help=_('The file to be included in the repo (path or glob)'), + ) + parser.add_argument( + "--no-checksum", + action="store_true", + default=False, + help=_("Don't use rsync checksums"), + ) + parser.add_argument( + "--archive-older", + type=int, + default=20, + help=_("Set maximum releases in repo before older ones are archived"), + ) # TODO add --with-btlog options = parser.parse_args() @@ -149,9 +220,11 @@ def main(): + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys') git_user_name = repo_git_base git_user_email = os.getenv('USER') + '@' + platform.node() - elif 'CIRCLE_REPOSITORY_URL' in os.environ \ - and 'CIRCLE_PROJECT_USERNAME' in os.environ \ - and 'CIRCLE_PROJECT_REPONAME' in os.environ: + elif ( + 'CIRCLE_REPOSITORY_URL' in os.environ + and 'CIRCLE_PROJECT_USERNAME' in os.environ + and 'CIRCLE_PROJECT_REPONAME' in os.environ + ): # we are in Circle CI repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME') + '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY) @@ -229,7 +302,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, config += "repo_description = 'Nightly builds from %s'\n" % git_user_email config += "archive_name = '%s'\n" % (repo_git_base + ' archive') config += "archive_url = '%s'\n" % (repo_base + '/archive') - config += "archive_description = 'Old nightly builds that have been archived.'\n" + config += ( + "archive_description = 'Old nightly builds that have been archived.'\n" + ) config += "archive_older = %i\n" % options.archive_older config += "servergitmirrors = '%s'\n" % servergitmirror config += "keystore = '%s'\n" % KEYSTORE_FILE @@ -252,25 +327,38 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, for f in files: if f.endswith('-debug.apk'): apkfilename = os.path.join(root, f) - logging.debug(_('Stripping mystery signature from {apkfilename}') - .format(apkfilename=apkfilename)) + logging.debug( + _('Stripping mystery signature from {apkfilename}').format( + apkfilename=apkfilename + ) + ) destapk = os.path.join(repodir, os.path.basename(f)) os.chmod(apkfilename, 0o644) - logging.debug(_('Resigning {apkfilename} with provided debug.keystore') - .format(apkfilename=os.path.basename(apkfilename))) + logging.debug( + _( + 'Resigning {apkfilename} with provided debug.keystore' + ).format(apkfilename=os.path.basename(apkfilename)) + ) common.apk_strip_v1_signatures(apkfilename, strip_manifest=True) common.sign_apk(apkfilename, destapk, KEY_ALIAS) if options.verbose: logging.debug(_('attempting bare SSH connection to test deploy key:')) try: - subprocess.check_call(['ssh', '-Tvi', ssh_private_key_file, - '-oIdentitiesOnly=yes', '-oStrictHostKeyChecking=no', - servergitmirror.split(':')[0]]) + subprocess.check_call( + [ + 'ssh', + '-Tvi', + ssh_private_key_file, + '-oIdentitiesOnly=yes', + '-oStrictHostKeyChecking=no', + servergitmirror.split(':')[0], + ] + ) except subprocess.CalledProcessError: pass - app_url = clone_url[:-len(NIGHTLY)] + app_url = clone_url[: -len(NIGHTLY)] template = dict() template['AuthorName'] = clone_url.split('/')[4] template['AuthorWebSite'] = '/'.join(clone_url.split('/')[:4]) @@ -282,9 +370,13 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, with open('template.yml', 'w') as fp: yaml.dump(template, fp) - subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], - cwd=repo_basedir) - common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/') + subprocess.check_call( + ['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], + cwd=repo_basedir, + ) + common.local_rsync( + options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/' + ) common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/') mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update app metadata") @@ -294,8 +386,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, cmd = ['fdroid', 'deploy', '--verbose', '--no-keep-git-mirror-archive'] subprocess.check_call(cmd, cwd=repo_basedir) except subprocess.CalledProcessError: - logging.error(_('cannot publish update, did you set the deploy key?') - + '\n' + deploy_key_url) + logging.error( + _('cannot publish update, did you set the deploy key?') + + '\n' + + deploy_key_url + ) sys.exit(1) if not options.keep_private_keys: @@ -326,8 +421,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, if options.show_secret_var: with open(options.keystore, 'rb') as fp: debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii') - print(_('\n{path} encoded for the DEBUG_KEYSTORE secret variable:') - .format(path=options.keystore)) + print( + _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format( + path=options.keystore + ) + ) print(debug_keystore) os.umask(umask) diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 07a1ea6e..12280628 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -76,11 +76,16 @@ def key_alias(appid): def read_fingerprints_from_keystore(): """Obtain a dictionary containing all singning-key fingerprints which are managed by F-Droid, grouped by appid.""" - env_vars = {'LC_ALL': 'C.UTF-8', - 'FDROID_KEY_STORE_PASS': config['keystorepass']} - cmd = [config['keytool'], '-list', - '-v', '-keystore', config['keystore'], - '-storepass:env', 'FDROID_KEY_STORE_PASS'] + env_vars = {'LC_ALL': 'C.UTF-8', 'FDROID_KEY_STORE_PASS': config['keystorepass']} + cmd = [ + config['keytool'], + '-list', + '-v', + '-keystore', + config['keystore'], + '-storepass:env', + 'FDROID_KEY_STORE_PASS', + ] if config['keystore'] == 'NONE': cmd += config['smartcardoptions'] p = FDroidPopen(cmd, envs=env_vars, output=False) @@ -116,8 +121,10 @@ def sign_sig_key_fingerprint_list(jar_file): cmd += config['smartcardoptions'] else: # smardcards never use -keypass cmd += '-keypass:env', 'FDROID_KEY_PASS' - env_vars = {'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config.get('keypass', "")} + env_vars = { + 'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config.get('keypass', ""), + } p = common.FDroidPopen(cmd, envs=env_vars) if p.returncode != 0: raise FDroidException("Failed to sign '{}'!".format(jar_file)) @@ -201,24 +208,44 @@ def create_key_if_not_existing(keyalias): """ # See if we already have a key for this application, and # if not generate one... - env_vars = {'LC_ALL': 'C.UTF-8', - 'FDROID_KEY_STORE_PASS': config['keystorepass'], - 'FDROID_KEY_PASS': config.get('keypass', "")} - cmd = [config['keytool'], '-list', - '-alias', keyalias, '-keystore', config['keystore'], - '-storepass:env', 'FDROID_KEY_STORE_PASS'] + env_vars = { + 'LC_ALL': 'C.UTF-8', + 'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config.get('keypass', ""), + } + cmd = [ + config['keytool'], + '-list', + '-alias', + keyalias, + '-keystore', + config['keystore'], + '-storepass:env', + 'FDROID_KEY_STORE_PASS', + ] if config['keystore'] == 'NONE': cmd += config['smartcardoptions'] p = FDroidPopen(cmd, envs=env_vars) if p.returncode != 0: logging.info("Key does not exist - generating...") - cmd = [config['keytool'], '-genkey', - '-keystore', config['keystore'], - '-alias', keyalias, - '-keyalg', 'RSA', '-keysize', '2048', - '-validity', '10000', - '-storepass:env', 'FDROID_KEY_STORE_PASS', - '-dname', config['keydname']] + cmd = [ + config['keytool'], + '-genkey', + '-keystore', + config['keystore'], + '-alias', + keyalias, + '-keyalg', + 'RSA', + '-keysize', + '2048', + '-validity', + '10000', + '-storepass:env', + 'FDROID_KEY_STORE_PASS', + '-dname', + config['keydname'], + ] if config['keystore'] == 'NONE': cmd += config['smartcardoptions'] else: @@ -235,11 +262,15 @@ def main(): global config, options # Parse command line... - parser = ArgumentParser(usage="%(prog)s [options] " - "[APPID[:VERCODE] [APPID[:VERCODE] ...]]") + parser = ArgumentParser( + usage="%(prog)s [options] " "[APPID[:VERCODE] [APPID[:VERCODE] ...]]" + ) common.setup_global_opts(parser) - parser.add_argument("appid", nargs='*', - help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) + parser.add_argument( + "appid", + nargs='*', + help=_("application ID with optional versionCode in the form APPID[:VERCODE]"), + ) metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W @@ -247,7 +278,9 @@ def main(): config = common.read_config(options) if not ('jarsigner' in config and 'keytool' in config): - logging.critical(_('Java JDK not found! Install in standard location or set java_paths!')) + logging.critical( + _('Java JDK not found! Install in standard location or set java_paths!') + ) sys.exit(1) common.assert_config_keystore(config) @@ -279,16 +312,21 @@ def main(): allapps = metadata.read_metadata() vercodes = common.read_pkg_args(options.appid, True) - common.get_metadata_files(vercodes) # only check appids + common.get_metadata_files(vercodes) # only check appids signed_apks = dict() generated_keys = dict() allaliases = check_for_key_collisions(allapps) - logging.info(ngettext('{0} app, {1} key aliases', - '{0} apps, {1} key aliases', len(allapps)).format(len(allapps), len(allaliases))) + logging.info( + ngettext( + '{0} app, {1} key aliases', '{0} apps, {1} key aliases', len(allapps) + ).format(len(allapps), len(allaliases)) + ) # Process any APKs or ZIPs that are waiting to be signed... - for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk')) - + glob.glob(os.path.join(unsigned_dir, '*.zip'))): + for apkfile in sorted( + glob.glob(os.path.join(unsigned_dir, '*.apk')) + + glob.glob(os.path.join(unsigned_dir, '*.zip')) + ): appid, vercode = common.publishednameinfo(apkfile) apkfilename = os.path.basename(apkfile) @@ -302,8 +340,9 @@ def main(): # There ought to be valid metadata for this app, otherwise why are we # trying to publish it? if appid not in allapps: - logging.error("Unexpected {0} found in unsigned directory" - .format(apkfilename)) + logging.error( + "Unexpected {0} found in unsigned directory".format(apkfilename) + ) sys.exit(1) app = allapps[appid] @@ -359,7 +398,9 @@ def main(): signature_file, _ignored, manifest, v2_files = signingfiles with open(signature_file, 'rb') as f: - devfp = common.signer_fingerprint_short(common.get_certificate(f.read())) + devfp = common.signer_fingerprint_short( + common.get_certificate(f.read()) + ) devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp) devsignedtmp = os.path.join(tmp_dir, devsigned) @@ -390,8 +431,7 @@ def main(): common.sign_apk(apkfile, signed_apk_path, keyalias) if appid not in signed_apks: signed_apks[appid] = [] - signed_apks[appid].append({"keyalias": keyalias, - "filename": apkfile}) + signed_apks[appid].append({"keyalias": keyalias, "filename": apkfile}) publish_source_tarball(apkfilename, unsigned_dir, output_dir) logging.info('Published ' + apkfilename) diff --git a/fdroidserver/signatures.py b/fdroidserver/signatures.py index 78f4bcd3..6d62cb07 100644 --- a/fdroidserver/signatures.py +++ b/fdroidserver/signatures.py @@ -89,8 +89,9 @@ def extract(options): def main(): parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument("APK", nargs='*', - help=_("signed APK, either a file-path or HTTPS URL.")) + parser.add_argument( + "APK", nargs='*', help=_("signed APK, either a file-path or HTTPS URL.") + ) parser.add_argument("--no-check-https", action="store_true", default=False) options = parser.parse_args() diff --git a/fdroidserver/signindex.py b/fdroidserver/signindex.py index 3b1f671e..a586c92b 100644 --- a/fdroidserver/signindex.py +++ b/fdroidserver/signindex.py @@ -41,10 +41,19 @@ def sign_jar(jar): but then Android < 4.3 would not be able to verify it. https://code.google.com/p/android/issues/detail?id=38321 """ - args = [config['jarsigner'], '-keystore', config['keystore'], - '-storepass:env', 'FDROID_KEY_STORE_PASS', - '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA', - jar, config['repo_keyalias']] + args = [ + config['jarsigner'], + '-keystore', + config['keystore'], + '-storepass:env', + 'FDROID_KEY_STORE_PASS', + '-digestalg', + 'SHA1', + '-sigalg', + 'SHA1withRSA', + jar, + config['repo_keyalias'], + ] if config['keystore'] == 'NONE': args += config['smartcardoptions'] else: # smardcards never use -keypass @@ -96,7 +105,9 @@ def main(): if 'jarsigner' not in config: raise FDroidException( - _('Java jarsigner not found! Install in standard location or set java_paths!') + _( + 'Java jarsigner not found! Install in standard location or set java_paths!' + ) ) repodirs = ['repo'] diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 84ac7928..2483a251 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -42,14 +42,18 @@ def get_clean_builder(serverdir): vagrantfile = os.path.join(serverdir, 'Vagrantfile') if not os.path.isfile(vagrantfile): with open(vagrantfile, 'w') as f: - f.write(textwrap.dedent("""\ + f.write( + textwrap.dedent( + """\ # generated file, do not change. Vagrant.configure("2") do |config| config.vm.box = "buildserver" config.vm.synced_folder ".", "/vagrant", disabled: true end - """)) + """ + ) + ) vm = get_build_vm(serverdir) logging.info('destroying buildserver before build') vm.destroy() @@ -119,25 +123,37 @@ def get_build_vm(srvdir, provider=None): if kvm_installed and vbox_installed: logging.debug('both kvm and vbox are installed.') elif kvm_installed: - logging.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'') + logging.debug( + 'libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'' + ) return LibvirtBuildVm(abssrvdir) elif vbox_installed: - logging.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'') + logging.debug( + 'virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'' + ) return VirtualboxBuildVm(abssrvdir) else: - logging.debug('could not confirm that either virtualbox or kvm/libvirt are installed') + logging.debug( + 'could not confirm that either virtualbox or kvm/libvirt are installed' + ) # try guessing provider from .../srvdir/.vagrant internals - vagrant_libvirt_path = os.path.join(abssrvdir, '.vagrant', 'machines', - 'default', 'libvirt') - has_libvirt_machine = isdir(vagrant_libvirt_path) \ - and len(os.listdir(vagrant_libvirt_path)) > 0 - vagrant_virtualbox_path = os.path.join(abssrvdir, '.vagrant', 'machines', - 'default', 'virtualbox') - has_vbox_machine = isdir(vagrant_virtualbox_path) \ - and len(os.listdir(vagrant_virtualbox_path)) > 0 + vagrant_libvirt_path = os.path.join( + abssrvdir, '.vagrant', 'machines', 'default', 'libvirt' + ) + has_libvirt_machine = ( + isdir(vagrant_libvirt_path) and len(os.listdir(vagrant_libvirt_path)) > 0 + ) + vagrant_virtualbox_path = os.path.join( + abssrvdir, '.vagrant', 'machines', 'default', 'virtualbox' + ) + has_vbox_machine = ( + isdir(vagrant_virtualbox_path) and len(os.listdir(vagrant_virtualbox_path)) > 0 + ) if has_libvirt_machine and has_vbox_machine: - logging.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'') + logging.info( + 'build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'' + ) return VirtualboxBuildVm(abssrvdir) elif has_libvirt_machine: logging.debug('build vm provider lookup found \'libvirt\'') @@ -149,12 +165,15 @@ def get_build_vm(srvdir, provider=None): # try guessing provider from available buildserver boxes available_boxes = [] import vagrant + boxes = vagrant.Vagrant().box_list() for box in boxes: if box.name == "buildserver": available_boxes.append(box.provider) if "libvirt" in available_boxes and "virtualbox" in available_boxes: - logging.info('basebox lookup found virtualbox and libvirt boxes, defaulting to \'virtualbox\'') + logging.info( + 'basebox lookup found virtualbox and libvirt boxes, defaulting to \'virtualbox\'' + ) return VirtualboxBuildVm(abssrvdir) elif "libvirt" in available_boxes: logging.info('\'libvirt\' buildserver box available, using that') @@ -171,7 +190,7 @@ class FDroidBuildVmException(FDroidException): pass -class FDroidBuildVm(): +class FDroidBuildVm: """Abstract base class for working with FDroids build-servers. Use the factory method `fdroidserver.vmtools.get_build_vm()` for @@ -188,11 +207,18 @@ class FDroidBuildVm(): self.vgrntfile = os.path.join(srvdir, 'Vagrantfile') self.srvuuid = self._vagrant_fetch_uuid() if not isdir(srvdir): - raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir)) + raise FDroidBuildVmException( + "Can not init vagrant, directory %s not present" % (srvdir) + ) if not isfile(self.vgrntfile): - raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile)) + raise FDroidBuildVmException( + "Can not init vagrant, '%s' not present" % (self.vgrntfile) + ) import vagrant - self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm) + + self.vgrnt = vagrant.Vagrant( + root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm + ) def up(self, provision=True): global lock @@ -302,7 +328,9 @@ class FDroidBuildVm(): """ boxfile = abspath(boxfile) if not isfile(boxfile): - raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile) + raise FDroidBuildVmException( + 'supplied boxfile \'%s\' does not exist', boxfile + ) self.vgrnt.box_add(boxname, abspath(boxfile), force=force) def box_remove(self, boxname): @@ -310,11 +338,13 @@ class FDroidBuildVm(): _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname]) except subprocess.CalledProcessError as e: logging.debug('tried removing box %s, but is did not exist: %s', boxname, e) - boxpath = os.path.join(expanduser('~'), '.vagrant', - self._vagrant_file_name(boxname)) + boxpath = os.path.join( + expanduser('~'), '.vagrant', self._vagrant_file_name(boxname) + ) if isdir(boxpath): - logging.info("attempting to remove box '%s' by deleting: %s", - boxname, boxpath) + logging.info( + "attempting to remove box '%s' by deleting: %s", boxname, boxpath + ) shutil.rmtree(boxpath) def sshinfo(self): @@ -325,11 +355,11 @@ class FDroidBuildVm(): A dictionary containing 'hostname', 'port', 'user' and 'idfile' """ import paramiko + try: sshconfig_path = os.path.join(self.srvdir, 'sshconfig') with open(sshconfig_path, 'wb') as fp: - fp.write(_check_output(['vagrant', 'ssh-config'], - cwd=self.srvdir)) + fp.write(_check_output(['vagrant', 'ssh-config'], cwd=self.srvdir)) vagranthost = 'default' # Host in ssh config file sshconfig = paramiko.SSHConfig() with open(sshconfig_path, 'r') as f: @@ -340,10 +370,12 @@ class FDroidBuildVm(): idfile = idfile[0] elif idfile.startswith('"') and idfile.endswith('"'): idfile = idfile[1:-1] - return {'hostname': sshconfig['hostname'], - 'port': int(sshconfig['port']), - 'user': sshconfig['user'], - 'idfile': idfile} + return { + 'hostname': sshconfig['hostname'], + 'port': int(sshconfig['port']), + 'user': sshconfig['user'], + 'idfile': idfile, + } except subprocess.CalledProcessError as e: raise FDroidBuildVmException("Error getting ssh config") from e @@ -411,21 +443,27 @@ class LibvirtBuildVm(FDroidBuildVm): # TODO use a libvirt storage pool to ensure the img file is readable if not os.access(imagepath, os.R_OK): logging.warning(_('Cannot read "{path}"!').format(path=imagepath)) - _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + _check_call( + ['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'] + ) shutil.copy2(imagepath, 'box.img') _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) - img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img']) + img_info_raw = _check_output( + ['qemu-img', 'info', '--output=json', 'box.img'] + ) img_info = json.loads(img_info_raw.decode('utf-8')) - metadata = {"provider": "libvirt", - "format": img_info['format'], - "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)), - } + metadata = { + "provider": "libvirt", + "format": img_info['format'], + "virtual_size": math.ceil(img_info['virtual-size'] / (1024.0 ** 3)), + } logging.debug('preparing metadata.json for box %s', output) with open('metadata.json', 'w') as fp: fp.write(json.dumps(metadata)) logging.debug('preparing Vagrantfile for box %s', output) - vagrantfile = textwrap.dedent("""\ + vagrantfile = textwrap.dedent( + """\ Vagrant.configure("2") do |config| config.ssh.username = "vagrant" config.ssh.password = "vagrant" @@ -440,11 +478,18 @@ class LibvirtBuildVm(FDroidBuildVm): libvirt.memory = {memory} end - end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])})) + end""".format_map( + { + 'memory': str(int(domainInfo[1] / 1024)), + 'cpus': str(domainInfo[3]), + } + ) + ) with open('Vagrantfile', 'w') as fp: fp.write(vagrantfile) try: import libarchive + with libarchive.file_writer(output, 'gnutar', 'gzip') as tar: logging.debug('adding files to box %s ...', output) tar.add_files('metadata.json', 'Vagrantfile', 'box.img') @@ -506,6 +551,7 @@ class LibvirtBuildVm(FDroidBuildVm): def snapshot_exists(self, snapshot_name): import libvirt + try: dom = self.conn.lookupByName(self.srvname) return dom.snapshotLookupByName(snapshot_name) is not None @@ -515,6 +561,7 @@ class LibvirtBuildVm(FDroidBuildVm): def snapshot_revert(self, snapshot_name): logging.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name) import libvirt + try: dom = self.conn.lookupByName(self.srvname) snap = dom.snapshotLookupByName(snapshot_name)