kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-20 13:50:12 +01:00
Code Activity

run black to reformat code that does not have WIP merge requests

Browse Source
This commit is contained in:
Hans-Christoph Steiner 2021-06-28 18:57:49 +02:00
parent 2e6cad57aa
commit 307cf8958c
No known key found for this signature in database
GPG Key ID: 3E177817BA1B9BFA
9 changed files with 519 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fdroidserver/__init__.py
View File

@ -1,4 +1,3 @@
import gettext import gettext
import glob import glob
import os import os
@ -8,7 +7,9 @@ import sys
# support running straight from git and standard installs # support running straight from git and standard installs
rootpaths = [ rootpaths = [
os.path.realpath(os.path.join(os.path.dirname(__file__), '..')), os.path.realpath(os.path.join(os.path.dirname(__file__), '..')),
os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share')), os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share')
),
os.path.join(sys.prefix, 'share'), os.path.join(sys.prefix, 'share'),
] ]

4
fdroidserver/gpgsign.py
View File

@ -59,7 +59,9 @@ def main():
signed = [] signed = []
for output_dir in repodirs: for output_dir in repodirs:
if not os.path.isdir(output_dir): if not os.path.isdir(output_dir):
raise FDroidException(_("Missing output directory") + " '" + output_dir + "'") raise FDroidException(
_("Missing output directory") + " '" + output_dir + "'"
)
# Process any apks that are waiting to be signed... # Process any apks that are waiting to be signed...
for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))): for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))):

129
fdroidserver/init.py
View File

@ -38,6 +38,7 @@ options = None
def disable_in_config(key, value): def disable_in_config(key, value):
"""Write a key/value to the local config.yml, then comment it out.""" """Write a key/value to the local config.yml, then comment it out."""
import yaml import yaml
with open('config.yml') as f: with open('config.yml') as f:
data = f.read() data = f.read()
pattern = r'\n[\s#]*' + key + r':.*' pattern = r'\n[\s#]*' + key + r':.*'
@ -54,16 +55,33 @@ def main():
# Parse command line... # Parse command line...
parser = ArgumentParser() parser = ArgumentParser()
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument("-d", "--distinguished-name", default=None, parser.add_argument(
help=_("X.509 'Distinguished Name' used when generating keys")) "-d",
parser.add_argument("--keystore", default=None, "--distinguished-name",
help=_("Path to the keystore for the repo signing key")) default=None,
parser.add_argument("--repo-keyalias", default=None, help=_("X.509 'Distinguished Name' used when generating keys"),
help=_("Alias of the repo signing key in the keystore")) )
parser.add_argument("--android-home", default=None, parser.add_argument(
help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)")) "--keystore",
parser.add_argument("--no-prompt", action="store_true", default=False, default=None,
help=_("Do not prompt for Android SDK path, just fail")) help=_("Path to the keystore for the repo signing key"),
)
parser.add_argument(
"--repo-keyalias",
default=None,
help=_("Alias of the repo signing key in the keystore"),
)
parser.add_argument(
"--android-home",
default=None,
help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)"),
)
parser.add_argument(
"--no-prompt",
action="store_true",
default=False,
help=_("Do not prompt for Android SDK path, just fail"),
)
options = parser.parse_args() options = parser.parse_args()
fdroiddir = os.getcwd() fdroiddir = os.getcwd()
@ -81,8 +99,9 @@ def main():
# and if the user leaves it blank, ignore and move on. # and if the user leaves it blank, ignore and move on.
default_sdk_path = '' default_sdk_path = ''
if sys.platform == 'win32' or sys.platform == 'cygwin': if sys.platform == 'win32' or sys.platform == 'cygwin':
p = os.path.join(os.getenv('USERPROFILE'), p = os.path.join(
'AppData', 'Local', 'Android', 'android-sdk') os.getenv('USERPROFILE'), 'AppData', 'Local', 'Android', 'android-sdk'
)
elif sys.platform == 'darwin': elif sys.platform == 'darwin':
# on OSX, Homebrew is common and has an easy path to detect # on OSX, Homebrew is common and has an easy path to detect
p = '/usr/local/opt/android-sdk' p = '/usr/local/opt/android-sdk'
@ -96,10 +115,13 @@ def main():
test_config['sdk_path'] = default_sdk_path test_config['sdk_path'] = default_sdk_path
if not common.test_sdk_exists(test_config): if not common.test_sdk_exists(test_config):
del(test_config['sdk_path']) del (test_config['sdk_path'])
while not options.no_prompt: while not options.no_prompt:
try: try:
s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path) s = input(
_('Enter the path to the Android SDK (%s) here:\n> ')
% default_sdk_path
)
except KeyboardInterrupt: except KeyboardInterrupt:
print('') print('')
sys.exit(1) sys.exit(1)
@ -112,8 +134,9 @@ def main():
default_sdk_path = '' default_sdk_path = ''
if test_config.get('sdk_path') and not common.test_sdk_exists(test_config): if test_config.get('sdk_path') and not common.test_sdk_exists(test_config):
raise FDroidException(_("Android SDK not found at {path}!") raise FDroidException(
.format(path=test_config['sdk_path'])) _("Android SDK not found at {path}!").format(path=test_config['sdk_path'])
)
if not os.path.exists('config.yml') and not os.path.exists('config.py'): if not os.path.exists('config.yml') and not os.path.exists('config.py'):
# 'metadata' and 'tmp' are created in fdroid # 'metadata' and 'tmp' are created in fdroid
@ -124,12 +147,14 @@ def main():
shutil.copyfile(example_config_yml, 'config.yml') shutil.copyfile(example_config_yml, 'config.yml')
else: else:
from pkg_resources import get_distribution from pkg_resources import get_distribution
versionstr = get_distribution('fdroidserver').version versionstr = get_distribution('fdroidserver').version
if not versionstr: if not versionstr:
versionstr = 'master' versionstr = 'master'
with open('config.yml', 'w') as fp: with open('config.yml', 'w') as fp:
fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/' fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/')
+ versionstr + '/examples/config.yml\n') fp.write(versionstr)
fp.write('/examples/config.yml\n')
os.chmod('config.yml', 0o0600) os.chmod('config.yml', 0o0600)
# If android_home is None, test_config['sdk_path'] will be used and # If android_home is None, test_config['sdk_path'] will be used and
# "$ANDROID_HOME" may be used if the env var is set up correctly. # "$ANDROID_HOME" may be used if the env var is set up correctly.
@ -138,7 +163,9 @@ def main():
if 'sdk_path' in test_config: if 'sdk_path' in test_config:
common.write_to_config(test_config, 'sdk_path', options.android_home) common.write_to_config(test_config, 'sdk_path', options.android_home)
else: else:
logging.warning('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...') logging.warning(
'Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...'
)
logging.info('Try running `fdroid init` in an empty directory.') logging.info('Try running `fdroid init` in an empty directory.')
raise FDroidException('Repository already exists.') raise FDroidException('Repository already exists.')
@ -162,8 +189,9 @@ def main():
else: else:
keystore = os.path.abspath(options.keystore) keystore = os.path.abspath(options.keystore)
if not os.path.exists(keystore): if not os.path.exists(keystore):
logging.info('"' + keystore logging.info(
+ '" does not exist, creating a new keystore there.') '"' + keystore + '" does not exist, creating a new keystore there.'
)
common.write_to_config(test_config, 'keystore', keystore) common.write_to_config(test_config, 'keystore', keystore)
repo_keyalias = None repo_keyalias = None
keydname = None keydname = None
@ -174,12 +202,19 @@ def main():
keydname = options.distinguished_name keydname = options.distinguished_name
common.write_to_config(test_config, 'keydname', keydname) common.write_to_config(test_config, 'keydname', keydname)
if keystore == 'NONE': # we're using a smartcard if keystore == 'NONE': # we're using a smartcard
common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default common.write_to_config(
test_config, 'repo_keyalias', '1'
) # seems to be the default
disable_in_config('keypass', 'never used with smartcard') disable_in_config('keypass', 'never used with smartcard')
common.write_to_config(test_config, 'smartcardoptions', common.write_to_config(
('-storetype PKCS11 ' test_config,
+ '-providerClass sun.security.pkcs11.SunPKCS11 ' 'smartcardoptions',
+ '-providerArg opensc-fdroid.cfg')) (
'-storetype PKCS11 '
+ '-providerClass sun.security.pkcs11.SunPKCS11 '
+ '-providerArg opensc-fdroid.cfg'
),
)
# find opensc-pkcs11.so # find opensc-pkcs11.so
if not os.path.exists('opensc-fdroid.cfg'): if not os.path.exists('opensc-fdroid.cfg'):
if os.path.exists('/usr/lib/opensc-pkcs11.so'): if os.path.exists('/usr/lib/opensc-pkcs11.so'):
@ -187,29 +222,43 @@ def main():
elif os.path.exists('/usr/lib64/opensc-pkcs11.so'): elif os.path.exists('/usr/lib64/opensc-pkcs11.so'):
opensc_so = '/usr/lib64/opensc-pkcs11.so' opensc_so = '/usr/lib64/opensc-pkcs11.so'
else: else:
files = glob.glob('/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so') files = glob.glob(
'/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so'
)
if len(files) > 0: if len(files) > 0:
opensc_so = files[0] opensc_so = files[0]
else: else:
opensc_so = '/usr/lib/opensc-pkcs11.so' opensc_so = '/usr/lib/opensc-pkcs11.so'
logging.warning('No OpenSC PKCS#11 module found, ' logging.warning(
+ 'install OpenSC then edit "opensc-fdroid.cfg"!') 'No OpenSC PKCS#11 module found, '
+ 'install OpenSC then edit "opensc-fdroid.cfg"!'
)
with open('opensc-fdroid.cfg', 'w') as f: with open('opensc-fdroid.cfg', 'w') as f:
f.write('name = OpenSC\nlibrary = ') f.write('name = OpenSC\nlibrary = ')
f.write(opensc_so) f.write(opensc_so)
f.write('\n') f.write('\n')
logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml.") logging.info(
logging.info("If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update " "Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml."
"--create-key'.") )
logging.info(
"If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update "
"--create-key'."
)
elif os.path.exists(keystore): elif os.path.exists(keystore):
to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname'] to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname']
if repo_keyalias: if repo_keyalias:
to_set.remove('repo_keyalias') to_set.remove('repo_keyalias')
if keydname: if keydname:
to_set.remove('keydname') to_set.remove('keydname')
logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore) logging.warning(
+ '\n' + _('Now set these in config.yml:') + ' ' '\n'
+ ', '.join(to_set) + '\n') + _('Using existing keystore "{path}"').format(path=keystore)
+ '\n'
+ _('Now set these in config.yml:')
+ ' '
+ ', '.join(to_set)
+ '\n'
)
else: else:
password = common.genpassword() password = common.genpassword()
c = dict(test_config) c = dict(test_config)
@ -229,11 +278,17 @@ def main():
msg += '\n ' + _('Keystore for signing key:\t') + keystore msg += '\n ' + _('Keystore for signing key:\t') + keystore
if repo_keyalias is not None: if repo_keyalias is not None:
msg += '\n Alias for key in store:\t' + repo_keyalias msg += '\n Alias for key in store:\t' + repo_keyalias
msg += '\n\n' + '''To complete the setup, add your APKs to "%s" msg += '\n\n'
msg += (
_(
'''To complete the setup, add your APKs to "%s"
then run "fdroid update -c; fdroid update". You might also want to edit then run "fdroid update -c; fdroid update". You might also want to edit
"config.yml" to set the URL, repo name, and more. You should also set up "config.yml" to set the URL, repo name, and more. You should also set up
a signing key (a temporary one might have been automatically generated). a signing key (a temporary one might have been automatically generated).
For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo
and https://f-droid.org/docs/Signing_Process''' % os.path.join(fdroiddir, 'repo') and https://f-droid.org/docs/Signing_Process'''
)
% os.path.join(fdroiddir, 'repo')
)
logging.info(msg) logging.info(msg)

133
fdroidserver/mirror.py
View File

@ -33,8 +33,15 @@ def _run_wget(path, urls):
with open(urls_file, 'w') as fp: with open(urls_file, 'w') as fp:
for url in urls: for url in urls:
fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename
subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"', subprocess.call(
'--input-file=' + urls_file]) [
'wget',
verbose,
'--continue',
'--user-agent="fdroid mirror"',
'--input-file=' + urls_file,
]
)
os.remove(urls_file) os.remove(urls_file)
@ -43,21 +50,47 @@ def main():
parser = ArgumentParser() parser = ArgumentParser()
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument("url", nargs='?', parser.add_argument(
help=_('Base URL to mirror, can include the index signing key ' "url",
+ 'using the query string: ?fingerprint=')) nargs='?',
parser.add_argument("--all", action='store_true', default=False, help=_(
help=_("Mirror the full repo and archive, all file types.")) 'Base URL to mirror, can include the index signing key '
parser.add_argument("--archive", action='store_true', default=False, + 'using the query string: ?fingerprint='
help=_("Also mirror the full archive section")) ),
parser.add_argument("--build-logs", action='store_true', default=False, )
help=_("Include the build logs in the mirror")) parser.add_argument(
parser.add_argument("--pgp-signatures", action='store_true', default=False, "--all",
help=_("Include the PGP signature .asc files in the mirror")) action='store_true',
parser.add_argument("--src-tarballs", action='store_true', default=False, default=False,
help=_("Include the source tarballs in the mirror")) help=_("Mirror the full repo and archive, all file types."),
parser.add_argument("--output-dir", default=None, )
help=_("The directory to write the mirror to")) parser.add_argument(
"--archive",
action='store_true',
default=False,
help=_("Also mirror the full archive section"),
)
parser.add_argument(
"--build-logs",
action='store_true',
default=False,
help=_("Include the build logs in the mirror"),
)
parser.add_argument(
"--pgp-signatures",
action='store_true',
default=False,
help=_("Include the PGP signature .asc files in the mirror"),
)
parser.add_argument(
"--src-tarballs",
action='store_true',
default=False,
help=_("Include the source tarballs in the mirror"),
)
parser.add_argument(
"--output-dir", default=None, help=_("The directory to write the mirror to")
)
options = parser.parse_args() options = parser.parse_args()
if options.all: if options.all:
@ -77,24 +110,31 @@ def main():
def _append_to_url_path(*args): def _append_to_url_path(*args):
"""Append the list of path components to URL, keeping the rest the same.""" """Append the list of path components to URL, keeping the rest the same."""
newpath = posixpath.join(path, *args) newpath = posixpath.join(path, *args)
return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment)) return urllib.parse.urlunparse(
(scheme, hostname, newpath, params, query, fragment)
)
if fingerprint: if fingerprint:
config = common.read_config(options) config = common.read_config(options)
if not ('jarsigner' in config or 'apksigner' in config): if not ('jarsigner' in config or 'apksigner' in config):
logging.error(_('Java JDK not found! Install in standard location or set java_paths!')) logging.error(
_('Java JDK not found! Install in standard location or set java_paths!')
)
sys.exit(1) sys.exit(1)
def _get_index(section, etag=None): def _get_index(section, etag=None):
url = _append_to_url_path(section) url = _append_to_url_path(section)
data, etag = index.download_repo_index(url, etag=etag) data, etag = index.download_repo_index(url, etag=etag)
return data, etag, _append_to_url_path(section, 'index-v1.jar') return data, etag, _append_to_url_path(section, 'index-v1.jar')
else: else:
def _get_index(section, etag=None): def _get_index(section, etag=None):
import io import io
import json import json
import zipfile import zipfile
from . import net from . import net
url = _append_to_url_path(section, 'index-v1.jar') url = _append_to_url_path(section, 'index-v1.jar')
content, etag = net.http_get(url) content, etag = net.http_get(url)
with zipfile.ZipFile(io.BytesIO(content)) as zip: with zipfile.ZipFile(io.BytesIO(content)) as zip:
@ -107,21 +147,30 @@ def main():
ip = ipaddress.ip_address(hostname) ip = ipaddress.ip_address(hostname)
except ValueError: except ValueError:
pass pass
if hostname == 'f-droid.org' \ if hostname == 'f-droid.org' or (
or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]): ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]
print(_('ERROR: this command should never be used to mirror f-droid.org!\n' ):
'A full mirror of f-droid.org requires more than 200GB.')) print(
_(
'ERROR: this command should never be used to mirror f-droid.org!\n'
'A full mirror of f-droid.org requires more than 200GB.'
)
)
sys.exit(1) sys.exit(1)
path = path.rstrip('/') path = path.rstrip('/')
if path.endswith('repo') or path.endswith('archive'): if path.endswith('repo') or path.endswith('archive'):
logging.warning(_('Do not include "{path}" in URL!') logging.warning(
.format(path=path.split('/')[-1])) _('Do not include "{path}" in URL!').format(path=path.split('/')[-1])
)
elif not path.endswith('fdroid'): elif not path.endswith('fdroid'):
logging.warning(_('{url} does not end with "fdroid", check the URL path!') logging.warning(
.format(url=options.url)) _('{url} does not end with "fdroid", check the URL path!').format(
url=options.url
)
)
icondirs = ['icons', ] icondirs = ['icons']
for density in update.screen_densities: for density in update.screen_densities:
icondirs.append('icons-' + density) icondirs.append('icons-' + density)
@ -134,7 +183,7 @@ def main():
if options.archive: if options.archive:
sections = ('repo', 'archive') sections = ('repo', 'archive')
else: else:
sections = ('repo', ) sections = ('repo',)
for section in sections: for section in sections:
sectiondir = os.path.join(basedir, section) sectiondir = os.path.join(basedir, section)
@ -152,23 +201,29 @@ def main():
for packageName, packageList in data['packages'].items(): for packageName, packageList in data['packages'].items():
for package in packageList: for package in packageList:
to_fetch = [] to_fetch = []
keys = ['apkName', ] keys = ['apkName']
if options.src_tarballs: if options.src_tarballs:
keys.append('srcname') keys.append('srcname')
for k in keys: for k in keys:
if k in package: if k in package:
to_fetch.append(package[k]) to_fetch.append(package[k])
elif k == 'apkName': elif k == 'apkName':
logging.error(_('{appid} is missing {name}') logging.error(
.format(appid=package['packageName'], name=k)) _('{appid} is missing {name}').format(
appid=package['packageName'], name=k
)
)
for f in to_fetch: for f in to_fetch:
if not os.path.exists(f) \ if not os.path.exists(f) or (
or (f.endswith('.apk') and os.path.getsize(f) != package['size']): f.endswith('.apk') and os.path.getsize(f) != package['size']
):
urls.append(_append_to_url_path(section, f)) urls.append(_append_to_url_path(section, f))
if options.pgp_signatures: if options.pgp_signatures:
urls.append(_append_to_url_path(section, f + '.asc')) urls.append(_append_to_url_path(section, f + '.asc'))
if options.build_logs and f.endswith('.apk'): if options.build_logs and f.endswith('.apk'):
urls.append(_append_to_url_path(section, f[:-4] + '.log.gz')) urls.append(
_append_to_url_path(section, f[:-4] + '.log.gz')
)
_run_wget(sectiondir, urls) _run_wget(sectiondir, urls)
@ -181,7 +236,7 @@ def main():
for k in update.GRAPHIC_NAMES: for k in update.GRAPHIC_NAMES:
f = d.get(k) f = d.get(k)
if f: if f:
filepath_tuple = components + (f, ) filepath_tuple = components + (f,)
urls.append(_append_to_url_path(*filepath_tuple)) urls.append(_append_to_url_path(*filepath_tuple))
_run_wget(os.path.join(basedir, *components), urls) _run_wget(os.path.join(basedir, *components), urls)
for k in update.SCREENSHOT_DIRS: for k in update.SCREENSHOT_DIRS:
@ -190,14 +245,16 @@ def main():
if filelist: if filelist:
components = (section, app['packageName'], locale, k) components = (section, app['packageName'], locale, k)
for f in filelist: for f in filelist:
filepath_tuple = components + (f, ) filepath_tuple = components + (f,)
urls.append(_append_to_url_path(*filepath_tuple)) urls.append(_append_to_url_path(*filepath_tuple))
_run_wget(os.path.join(basedir, *components), urls) _run_wget(os.path.join(basedir, *components), urls)
urls = dict() urls = dict()
for app in data['apps']: for app in data['apps']:
if 'icon' not in app: if 'icon' not in app:
logging.error(_('no "icon" in {appid}').format(appid=app['packageName'])) logging.error(
_('no "icon" in {appid}').format(appid=app['packageName'])
)
continue continue
icon = app['icon'] icon = app['icon']
for icondir in icondirs: for icondir in icondirs:

196
fdroidserver/nightly.py
View File

@ -54,27 +54,72 @@ def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE):
p12 = os.path.join(tmp_dir, '.keystore.p12') p12 = os.path.join(tmp_dir, '.keystore.p12')
_config = dict() _config = dict()
common.fill_config_defaults(_config) common.fill_config_defaults(_config)
subprocess.check_call([_config['keytool'], '-importkeystore', subprocess.check_call(
'-srckeystore', keystore, '-srcalias', KEY_ALIAS, [
'-srcstorepass', PASSWORD, '-srckeypass', PASSWORD, _config['keytool'],
'-destkeystore', p12, '-destalias', KEY_ALIAS, '-importkeystore',
'-deststorepass', PASSWORD, '-destkeypass', PASSWORD, '-srckeystore',
'-deststoretype', 'PKCS12'], keystore,
env={'LC_ALL': 'C.UTF-8'}) '-srcalias',
subprocess.check_call(['openssl', 'pkcs12', '-in', p12, '-out', key_pem, KEY_ALIAS,
'-passin', 'pass:' + PASSWORD, '-passout', 'pass:' + PASSWORD], '-srcstorepass',
env={'LC_ALL': 'C.UTF-8'}) PASSWORD,
subprocess.check_call(['openssl', 'rsa', '-in', key_pem, '-out', privkey, '-srckeypass',
'-passin', 'pass:' + PASSWORD], PASSWORD,
env={'LC_ALL': 'C.UTF-8'}) '-destkeystore',
p12,
'-destalias',
KEY_ALIAS,
'-deststorepass',
PASSWORD,
'-destkeypass',
PASSWORD,
'-deststoretype',
'PKCS12',
],
env={'LC_ALL': 'C.UTF-8'},
)
subprocess.check_call(
[
'openssl',
'pkcs12',
'-in',
p12,
'-out',
key_pem,
'-passin',
'pass:' + PASSWORD,
'-passout',
'pass:' + PASSWORD,
],
env={'LC_ALL': 'C.UTF-8'},
)
subprocess.check_call(
[
'openssl',
'rsa',
'-in',
key_pem,
'-out',
privkey,
'-passin',
'pass:' + PASSWORD,
],
env={'LC_ALL': 'C.UTF-8'},
)
os.remove(key_pem) os.remove(key_pem)
os.remove(p12) os.remove(p12)
os.chmod(privkey, 0o600) # os.umask() should cover this, but just in case os.chmod(privkey, 0o600) # os.umask() should cover this, but just in case
rsakey = paramiko.RSAKey.from_private_key_file(privkey) rsakey = paramiko.RSAKey.from_private_key_file(privkey)
fingerprint = base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()).decode('ascii').rstrip('=') fingerprint = (
ssh_private_key_file = os.path.join(tmp_dir, 'debug_keystore_' base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest())
+ fingerprint.replace('/', '_') + '_id_rsa') .decode('ascii')
.rstrip('=')
)
ssh_private_key_file = os.path.join(
tmp_dir, 'debug_keystore_' + fingerprint.replace('/', '_') + '_id_rsa'
)
shutil.move(privkey, ssh_private_key_file) shutil.move(privkey, ssh_private_key_file)
pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file
@ -90,20 +135,46 @@ def main():
parser = ArgumentParser() parser = ArgumentParser()
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument("--keystore", default=KEYSTORE_FILE, parser.add_argument(
help=_("Specify which debug keystore file to use.")) "--keystore",
parser.add_argument("--show-secret-var", action="store_true", default=False, default=KEYSTORE_FILE,
help=_("Print the secret variable to the terminal for easy copy/paste")) help=_("Specify which debug keystore file to use."),
parser.add_argument("--keep-private-keys", action="store_true", default=False, )
help=_("Do not remove the private keys generated from the keystore")) parser.add_argument(
parser.add_argument("--no-deploy", action="store_true", default=False, "--show-secret-var",
help=_("Do not deploy the new files to the repo")) action="store_true",
parser.add_argument("--file", default='app/build/outputs/apk/*.apk', default=False,
help=_('The file to be included in the repo (path or glob)')) help=_("Print the secret variable to the terminal for easy copy/paste"),
parser.add_argument("--no-checksum", action="store_true", default=False, )
help=_("Don't use rsync checksums")) parser.add_argument(
parser.add_argument("--archive-older", type=int, default=20, "--keep-private-keys",
help=_("Set maximum releases in repo before older ones are archived")) action="store_true",
default=False,
help=_("Do not remove the private keys generated from the keystore"),
)
parser.add_argument(
"--no-deploy",
action="store_true",
default=False,
help=_("Do not deploy the new files to the repo"),
)
parser.add_argument(
"--file",
default='app/build/outputs/apk/*.apk',
help=_('The file to be included in the repo (path or glob)'),
)
parser.add_argument(
"--no-checksum",
action="store_true",
default=False,
help=_("Don't use rsync checksums"),
)
parser.add_argument(
"--archive-older",
type=int,
default=20,
help=_("Set maximum releases in repo before older ones are archived"),
)
# TODO add --with-btlog # TODO add --with-btlog
options = parser.parse_args() options = parser.parse_args()
@ -149,9 +220,11 @@ def main():
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys') + '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys')
git_user_name = repo_git_base git_user_name = repo_git_base
git_user_email = os.getenv('USER') + '@' + platform.node() git_user_email = os.getenv('USER') + '@' + platform.node()
elif 'CIRCLE_REPOSITORY_URL' in os.environ \ elif (
and 'CIRCLE_PROJECT_USERNAME' in os.environ \ 'CIRCLE_REPOSITORY_URL' in os.environ
and 'CIRCLE_PROJECT_REPONAME' in os.environ: and 'CIRCLE_PROJECT_USERNAME' in os.environ
and 'CIRCLE_PROJECT_REPONAME' in os.environ
):
# we are in Circle CI # we are in Circle CI
repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME') repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME')
+ '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY) + '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY)
@ -229,7 +302,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
config += "repo_description = 'Nightly builds from %s'\n" % git_user_email config += "repo_description = 'Nightly builds from %s'\n" % git_user_email
config += "archive_name = '%s'\n" % (repo_git_base + ' archive') config += "archive_name = '%s'\n" % (repo_git_base + ' archive')
config += "archive_url = '%s'\n" % (repo_base + '/archive') config += "archive_url = '%s'\n" % (repo_base + '/archive')
config += "archive_description = 'Old nightly builds that have been archived.'\n" config += (
"archive_description = 'Old nightly builds that have been archived.'\n"
)
config += "archive_older = %i\n" % options.archive_older config += "archive_older = %i\n" % options.archive_older
config += "servergitmirrors = '%s'\n" % servergitmirror config += "servergitmirrors = '%s'\n" % servergitmirror
config += "keystore = '%s'\n" % KEYSTORE_FILE config += "keystore = '%s'\n" % KEYSTORE_FILE
@ -252,25 +327,38 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
for f in files: for f in files:
if f.endswith('-debug.apk'): if f.endswith('-debug.apk'):
apkfilename = os.path.join(root, f) apkfilename = os.path.join(root, f)
logging.debug(_('Stripping mystery signature from {apkfilename}') logging.debug(
.format(apkfilename=apkfilename)) _('Stripping mystery signature from {apkfilename}').format(
apkfilename=apkfilename
)
)
destapk = os.path.join(repodir, os.path.basename(f)) destapk = os.path.join(repodir, os.path.basename(f))
os.chmod(apkfilename, 0o644) os.chmod(apkfilename, 0o644)
logging.debug(_('Resigning {apkfilename} with provided debug.keystore') logging.debug(
.format(apkfilename=os.path.basename(apkfilename))) _(
'Resigning {apkfilename} with provided debug.keystore'
).format(apkfilename=os.path.basename(apkfilename))
)
common.apk_strip_v1_signatures(apkfilename, strip_manifest=True) common.apk_strip_v1_signatures(apkfilename, strip_manifest=True)
common.sign_apk(apkfilename, destapk, KEY_ALIAS) common.sign_apk(apkfilename, destapk, KEY_ALIAS)
if options.verbose: if options.verbose:
logging.debug(_('attempting bare SSH connection to test deploy key:')) logging.debug(_('attempting bare SSH connection to test deploy key:'))
try: try:
subprocess.check_call(['ssh', '-Tvi', ssh_private_key_file, subprocess.check_call(
'-oIdentitiesOnly=yes', '-oStrictHostKeyChecking=no', [
servergitmirror.split(':')[0]]) 'ssh',
'-Tvi',
ssh_private_key_file,
'-oIdentitiesOnly=yes',
'-oStrictHostKeyChecking=no',
servergitmirror.split(':')[0],
]
)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
pass pass
app_url = clone_url[:-len(NIGHTLY)] app_url = clone_url[: -len(NIGHTLY)]
template = dict() template = dict()
template['AuthorName'] = clone_url.split('/')[4] template['AuthorName'] = clone_url.split('/')[4]
template['AuthorWebSite'] = '/'.join(clone_url.split('/')[:4]) template['AuthorWebSite'] = '/'.join(clone_url.split('/')[:4])
@ -282,9 +370,13 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
with open('template.yml', 'w') as fp: with open('template.yml', 'w') as fp:
yaml.dump(template, fp) yaml.dump(template, fp)
subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], subprocess.check_call(
cwd=repo_basedir) ['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'],
common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/') cwd=repo_basedir,
)
common.local_rsync(
options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
)
common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/') common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
mirror_git_repo.git.add(all=True) mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update app metadata") mirror_git_repo.index.commit("update app metadata")
@ -294,8 +386,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
cmd = ['fdroid', 'deploy', '--verbose', '--no-keep-git-mirror-archive'] cmd = ['fdroid', 'deploy', '--verbose', '--no-keep-git-mirror-archive']
subprocess.check_call(cmd, cwd=repo_basedir) subprocess.check_call(cmd, cwd=repo_basedir)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
logging.error(_('cannot publish update, did you set the deploy key?') logging.error(
+ '\n' + deploy_key_url) _('cannot publish update, did you set the deploy key?')
+ '\n'
+ deploy_key_url
)
sys.exit(1) sys.exit(1)
if not options.keep_private_keys: if not options.keep_private_keys:
@ -326,8 +421,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
if options.show_secret_var: if options.show_secret_var:
with open(options.keystore, 'rb') as fp: with open(options.keystore, 'rb') as fp:
debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii') debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii')
print(_('\n{path} encoded for the DEBUG_KEYSTORE secret variable:') print(
.format(path=options.keystore)) _('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format(
path=options.keystore
)
)
print(debug_keystore) print(debug_keystore)
os.umask(umask) os.umask(umask)

110
fdroidserver/publish.py
View File

@ -76,11 +76,16 @@ def key_alias(appid):
def read_fingerprints_from_keystore(): def read_fingerprints_from_keystore():
"""Obtain a dictionary containing all singning-key fingerprints which are managed by F-Droid, grouped by appid.""" """Obtain a dictionary containing all singning-key fingerprints which are managed by F-Droid, grouped by appid."""
env_vars = {'LC_ALL': 'C.UTF-8', env_vars = {'LC_ALL': 'C.UTF-8', 'FDROID_KEY_STORE_PASS': config['keystorepass']}
'FDROID_KEY_STORE_PASS': config['keystorepass']} cmd = [
cmd = [config['keytool'], '-list', config['keytool'],
'-v', '-keystore', config['keystore'], '-list',
'-storepass:env', 'FDROID_KEY_STORE_PASS'] '-v',
'-keystore',
config['keystore'],
'-storepass:env',
'FDROID_KEY_STORE_PASS',
]
if config['keystore'] == 'NONE': if config['keystore'] == 'NONE':
cmd += config['smartcardoptions'] cmd += config['smartcardoptions']
p = FDroidPopen(cmd, envs=env_vars, output=False) p = FDroidPopen(cmd, envs=env_vars, output=False)
@ -116,8 +121,10 @@ def sign_sig_key_fingerprint_list(jar_file):
cmd += config['smartcardoptions'] cmd += config['smartcardoptions']
else: # smardcards never use -keypass else: # smardcards never use -keypass
cmd += '-keypass:env', 'FDROID_KEY_PASS' cmd += '-keypass:env', 'FDROID_KEY_PASS'
env_vars = {'FDROID_KEY_STORE_PASS': config['keystorepass'], env_vars = {
'FDROID_KEY_PASS': config.get('keypass', "")} 'FDROID_KEY_STORE_PASS': config['keystorepass'],
'FDROID_KEY_PASS': config.get('keypass', ""),
}
p = common.FDroidPopen(cmd, envs=env_vars) p = common.FDroidPopen(cmd, envs=env_vars)
if p.returncode != 0: if p.returncode != 0:
raise FDroidException("Failed to sign '{}'!".format(jar_file)) raise FDroidException("Failed to sign '{}'!".format(jar_file))
@ -201,24 +208,44 @@ def create_key_if_not_existing(keyalias):
""" """
# See if we already have a key for this application, and # See if we already have a key for this application, and
# if not generate one... # if not generate one...
env_vars = {'LC_ALL': 'C.UTF-8', env_vars = {
'FDROID_KEY_STORE_PASS': config['keystorepass'], 'LC_ALL': 'C.UTF-8',
'FDROID_KEY_PASS': config.get('keypass', "")} 'FDROID_KEY_STORE_PASS': config['keystorepass'],
cmd = [config['keytool'], '-list', 'FDROID_KEY_PASS': config.get('keypass', ""),
'-alias', keyalias, '-keystore', config['keystore'], }
'-storepass:env', 'FDROID_KEY_STORE_PASS'] cmd = [
config['keytool'],
'-list',
'-alias',
keyalias,
'-keystore',
config['keystore'],
'-storepass:env',
'FDROID_KEY_STORE_PASS',
]
if config['keystore'] == 'NONE': if config['keystore'] == 'NONE':
cmd += config['smartcardoptions'] cmd += config['smartcardoptions']
p = FDroidPopen(cmd, envs=env_vars) p = FDroidPopen(cmd, envs=env_vars)
if p.returncode != 0: if p.returncode != 0:
logging.info("Key does not exist - generating...") logging.info("Key does not exist - generating...")
cmd = [config['keytool'], '-genkey', cmd = [
'-keystore', config['keystore'], config['keytool'],
'-alias', keyalias, '-genkey',
'-keyalg', 'RSA', '-keysize', '2048', '-keystore',
'-validity', '10000', config['keystore'],
'-storepass:env', 'FDROID_KEY_STORE_PASS', '-alias',
'-dname', config['keydname']] keyalias,
'-keyalg',
'RSA',
'-keysize',
'2048',
'-validity',
'10000',
'-storepass:env',
'FDROID_KEY_STORE_PASS',
'-dname',
config['keydname'],
]
if config['keystore'] == 'NONE': if config['keystore'] == 'NONE':
cmd += config['smartcardoptions'] cmd += config['smartcardoptions']
else: else:
@ -235,11 +262,15 @@ def main():
global config, options global config, options
# Parse command line... # Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] " parser = ArgumentParser(
"[APPID[:VERCODE] [APPID[:VERCODE] ...]]") usage="%(prog)s [options] " "[APPID[:VERCODE] [APPID[:VERCODE] ...]]"
)
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', parser.add_argument(
help=_("application ID with optional versionCode in the form APPID[:VERCODE]")) "appid",
nargs='*',
help=_("application ID with optional versionCode in the form APPID[:VERCODE]"),
)
metadata.add_metadata_arguments(parser) metadata.add_metadata_arguments(parser)
options = parser.parse_args() options = parser.parse_args()
metadata.warnings_action = options.W metadata.warnings_action = options.W
@ -247,7 +278,9 @@ def main():
config = common.read_config(options) config = common.read_config(options)
if not ('jarsigner' in config and 'keytool' in config): if not ('jarsigner' in config and 'keytool' in config):
logging.critical(_('Java JDK not found! Install in standard location or set java_paths!')) logging.critical(
_('Java JDK not found! Install in standard location or set java_paths!')
)
sys.exit(1) sys.exit(1)
common.assert_config_keystore(config) common.assert_config_keystore(config)
@ -279,16 +312,21 @@ def main():
allapps = metadata.read_metadata() allapps = metadata.read_metadata()
vercodes = common.read_pkg_args(options.appid, True) vercodes = common.read_pkg_args(options.appid, True)
common.get_metadata_files(vercodes) # only check appids common.get_metadata_files(vercodes) # only check appids
signed_apks = dict() signed_apks = dict()
generated_keys = dict() generated_keys = dict()
allaliases = check_for_key_collisions(allapps) allaliases = check_for_key_collisions(allapps)
logging.info(ngettext('{0} app, {1} key aliases', logging.info(
'{0} apps, {1} key aliases', len(allapps)).format(len(allapps), len(allaliases))) ngettext(
'{0} app, {1} key aliases', '{0} apps, {1} key aliases', len(allapps)
).format(len(allapps), len(allaliases))
)
# Process any APKs or ZIPs that are waiting to be signed... # Process any APKs or ZIPs that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk')) for apkfile in sorted(
+ glob.glob(os.path.join(unsigned_dir, '*.zip'))): glob.glob(os.path.join(unsigned_dir, '*.apk'))
+ glob.glob(os.path.join(unsigned_dir, '*.zip'))
):
appid, vercode = common.publishednameinfo(apkfile) appid, vercode = common.publishednameinfo(apkfile)
apkfilename = os.path.basename(apkfile) apkfilename = os.path.basename(apkfile)
@ -302,8 +340,9 @@ def main():
# There ought to be valid metadata for this app, otherwise why are we # There ought to be valid metadata for this app, otherwise why are we
# trying to publish it? # trying to publish it?
if appid not in allapps: if appid not in allapps:
logging.error("Unexpected {0} found in unsigned directory" logging.error(
.format(apkfilename)) "Unexpected {0} found in unsigned directory".format(apkfilename)
)
sys.exit(1) sys.exit(1)
app = allapps[appid] app = allapps[appid]
@ -359,7 +398,9 @@ def main():
signature_file, _ignored, manifest, v2_files = signingfiles signature_file, _ignored, manifest, v2_files = signingfiles
with open(signature_file, 'rb') as f: with open(signature_file, 'rb') as f:
devfp = common.signer_fingerprint_short(common.get_certificate(f.read())) devfp = common.signer_fingerprint_short(
common.get_certificate(f.read())
)
devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp) devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp)
devsignedtmp = os.path.join(tmp_dir, devsigned) devsignedtmp = os.path.join(tmp_dir, devsigned)
@ -390,8 +431,7 @@ def main():
common.sign_apk(apkfile, signed_apk_path, keyalias) common.sign_apk(apkfile, signed_apk_path, keyalias)
if appid not in signed_apks: if appid not in signed_apks:
signed_apks[appid] = [] signed_apks[appid] = []
signed_apks[appid].append({"keyalias": keyalias, signed_apks[appid].append({"keyalias": keyalias, "filename": apkfile})
"filename": apkfile})
publish_source_tarball(apkfilename, unsigned_dir, output_dir) publish_source_tarball(apkfilename, unsigned_dir, output_dir)
logging.info('Published ' + apkfilename) logging.info('Published ' + apkfilename)

5
fdroidserver/signatures.py
View File

@ -89,8 +89,9 @@ def extract(options):
def main(): def main():
parser = ArgumentParser() parser = ArgumentParser()
common.setup_global_opts(parser) common.setup_global_opts(parser)
parser.add_argument("APK", nargs='*', parser.add_argument(
help=_("signed APK, either a file-path or HTTPS URL.")) "APK", nargs='*', help=_("signed APK, either a file-path or HTTPS URL.")
)
parser.add_argument("--no-check-https", action="store_true", default=False) parser.add_argument("--no-check-https", action="store_true", default=False)
options = parser.parse_args() options = parser.parse_args()

21
fdroidserver/signindex.py
View File

@ -41,10 +41,19 @@ def sign_jar(jar):
but then Android < 4.3 would not be able to verify it. but then Android < 4.3 would not be able to verify it.
https://code.google.com/p/android/issues/detail?id=38321 https://code.google.com/p/android/issues/detail?id=38321
""" """
args = [config['jarsigner'], '-keystore', config['keystore'], args = [
'-storepass:env', 'FDROID_KEY_STORE_PASS', config['jarsigner'],
'-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA', '-keystore',
jar, config['repo_keyalias']] config['keystore'],
'-storepass:env',
'FDROID_KEY_STORE_PASS',
'-digestalg',
'SHA1',
'-sigalg',
'SHA1withRSA',
jar,
config['repo_keyalias'],
]
if config['keystore'] == 'NONE': if config['keystore'] == 'NONE':
args += config['smartcardoptions'] args += config['smartcardoptions']
else: # smardcards never use -keypass else: # smardcards never use -keypass
@ -96,7 +105,9 @@ def main():
if 'jarsigner' not in config: if 'jarsigner' not in config:
raise FDroidException( raise FDroidException(
_('Java jarsigner not found! Install in standard location or set java_paths!') _(
'Java jarsigner not found! Install in standard location or set java_paths!'
)
) )
repodirs = ['repo'] repodirs = ['repo']

123
fdroidserver/vmtools.py
View File

@ -42,14 +42,18 @@ def get_clean_builder(serverdir):
vagrantfile = os.path.join(serverdir, 'Vagrantfile') vagrantfile = os.path.join(serverdir, 'Vagrantfile')
if not os.path.isfile(vagrantfile): if not os.path.isfile(vagrantfile):
with open(vagrantfile, 'w') as f: with open(vagrantfile, 'w') as f:
f.write(textwrap.dedent("""\ f.write(
textwrap.dedent(
"""\
# generated file, do not change. # generated file, do not change.
Vagrant.configure("2") do |config| Vagrant.configure("2") do |config|
config.vm.box = "buildserver" config.vm.box = "buildserver"
config.vm.synced_folder ".", "/vagrant", disabled: true config.vm.synced_folder ".", "/vagrant", disabled: true
end end
""")) """
)
)
vm = get_build_vm(serverdir) vm = get_build_vm(serverdir)
logging.info('destroying buildserver before build') logging.info('destroying buildserver before build')
vm.destroy() vm.destroy()
@ -119,25 +123,37 @@ def get_build_vm(srvdir, provider=None):
if kvm_installed and vbox_installed: if kvm_installed and vbox_installed:
logging.debug('both kvm and vbox are installed.') logging.debug('both kvm and vbox are installed.')
elif kvm_installed: elif kvm_installed:
logging.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'') logging.debug(
'libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\''
)
return LibvirtBuildVm(abssrvdir) return LibvirtBuildVm(abssrvdir)
elif vbox_installed: elif vbox_installed:
logging.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'') logging.debug(
'virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\''
)
return VirtualboxBuildVm(abssrvdir) return VirtualboxBuildVm(abssrvdir)
else: else:
logging.debug('could not confirm that either virtualbox or kvm/libvirt are installed') logging.debug(
'could not confirm that either virtualbox or kvm/libvirt are installed'
)
# try guessing provider from .../srvdir/.vagrant internals # try guessing provider from .../srvdir/.vagrant internals
vagrant_libvirt_path = os.path.join(abssrvdir, '.vagrant', 'machines', vagrant_libvirt_path = os.path.join(
'default', 'libvirt') abssrvdir, '.vagrant', 'machines', 'default', 'libvirt'
has_libvirt_machine = isdir(vagrant_libvirt_path) \ )
and len(os.listdir(vagrant_libvirt_path)) > 0 has_libvirt_machine = (
vagrant_virtualbox_path = os.path.join(abssrvdir, '.vagrant', 'machines', isdir(vagrant_libvirt_path) and len(os.listdir(vagrant_libvirt_path)) > 0
'default', 'virtualbox') )
has_vbox_machine = isdir(vagrant_virtualbox_path) \ vagrant_virtualbox_path = os.path.join(
and len(os.listdir(vagrant_virtualbox_path)) > 0 abssrvdir, '.vagrant', 'machines', 'default', 'virtualbox'
)
has_vbox_machine = (
isdir(vagrant_virtualbox_path) and len(os.listdir(vagrant_virtualbox_path)) > 0
)
if has_libvirt_machine and has_vbox_machine: if has_libvirt_machine and has_vbox_machine:
logging.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'') logging.info(
'build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\''
)
return VirtualboxBuildVm(abssrvdir) return VirtualboxBuildVm(abssrvdir)
elif has_libvirt_machine: elif has_libvirt_machine:
logging.debug('build vm provider lookup found \'libvirt\'') logging.debug('build vm provider lookup found \'libvirt\'')
@ -149,12 +165,15 @@ def get_build_vm(srvdir, provider=None):
# try guessing provider from available buildserver boxes # try guessing provider from available buildserver boxes
available_boxes = [] available_boxes = []
import vagrant import vagrant
boxes = vagrant.Vagrant().box_list() boxes = vagrant.Vagrant().box_list()
for box in boxes: for box in boxes:
if box.name == "buildserver": if box.name == "buildserver":
available_boxes.append(box.provider) available_boxes.append(box.provider)
if "libvirt" in available_boxes and "virtualbox" in available_boxes: if "libvirt" in available_boxes and "virtualbox" in available_boxes:
logging.info('basebox lookup found virtualbox and libvirt boxes, defaulting to \'virtualbox\'') logging.info(
'basebox lookup found virtualbox and libvirt boxes, defaulting to \'virtualbox\''
)
return VirtualboxBuildVm(abssrvdir) return VirtualboxBuildVm(abssrvdir)
elif "libvirt" in available_boxes: elif "libvirt" in available_boxes:
logging.info('\'libvirt\' buildserver box available, using that') logging.info('\'libvirt\' buildserver box available, using that')
@ -171,7 +190,7 @@ class FDroidBuildVmException(FDroidException):
pass pass
class FDroidBuildVm(): class FDroidBuildVm:
"""Abstract base class for working with FDroids build-servers. """Abstract base class for working with FDroids build-servers.
Use the factory method `fdroidserver.vmtools.get_build_vm()` for Use the factory method `fdroidserver.vmtools.get_build_vm()` for
@ -188,11 +207,18 @@ class FDroidBuildVm():
self.vgrntfile = os.path.join(srvdir, 'Vagrantfile') self.vgrntfile = os.path.join(srvdir, 'Vagrantfile')
self.srvuuid = self._vagrant_fetch_uuid() self.srvuuid = self._vagrant_fetch_uuid()
if not isdir(srvdir): if not isdir(srvdir):
raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir)) raise FDroidBuildVmException(
"Can not init vagrant, directory %s not present" % (srvdir)
)
if not isfile(self.vgrntfile): if not isfile(self.vgrntfile):
raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile)) raise FDroidBuildVmException(
"Can not init vagrant, '%s' not present" % (self.vgrntfile)
)
import vagrant import vagrant
self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
self.vgrnt = vagrant.Vagrant(
root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm
)
def up(self, provision=True): def up(self, provision=True):
global lock global lock
@ -302,7 +328,9 @@ class FDroidBuildVm():
""" """
boxfile = abspath(boxfile) boxfile = abspath(boxfile)
if not isfile(boxfile): if not isfile(boxfile):
raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile) raise FDroidBuildVmException(
'supplied boxfile \'%s\' does not exist', boxfile
)
self.vgrnt.box_add(boxname, abspath(boxfile), force=force) self.vgrnt.box_add(boxname, abspath(boxfile), force=force)
def box_remove(self, boxname): def box_remove(self, boxname):
@ -310,11 +338,13 @@ class FDroidBuildVm():
_check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname]) _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
logging.debug('tried removing box %s, but is did not exist: %s', boxname, e) logging.debug('tried removing box %s, but is did not exist: %s', boxname, e)
boxpath = os.path.join(expanduser('~'), '.vagrant', boxpath = os.path.join(
self._vagrant_file_name(boxname)) expanduser('~'), '.vagrant', self._vagrant_file_name(boxname)
)
if isdir(boxpath): if isdir(boxpath):
logging.info("attempting to remove box '%s' by deleting: %s", logging.info(
boxname, boxpath) "attempting to remove box '%s' by deleting: %s", boxname, boxpath
)
shutil.rmtree(boxpath) shutil.rmtree(boxpath)
def sshinfo(self): def sshinfo(self):
@ -325,11 +355,11 @@ class FDroidBuildVm():
A dictionary containing 'hostname', 'port', 'user' and 'idfile' A dictionary containing 'hostname', 'port', 'user' and 'idfile'
""" """
import paramiko import paramiko
try: try:
sshconfig_path = os.path.join(self.srvdir, 'sshconfig') sshconfig_path = os.path.join(self.srvdir, 'sshconfig')
with open(sshconfig_path, 'wb') as fp: with open(sshconfig_path, 'wb') as fp:
fp.write(_check_output(['vagrant', 'ssh-config'], fp.write(_check_output(['vagrant', 'ssh-config'], cwd=self.srvdir))
cwd=self.srvdir))
vagranthost = 'default' # Host in ssh config file vagranthost = 'default' # Host in ssh config file
sshconfig = paramiko.SSHConfig() sshconfig = paramiko.SSHConfig()
with open(sshconfig_path, 'r') as f: with open(sshconfig_path, 'r') as f:
@ -340,10 +370,12 @@ class FDroidBuildVm():
idfile = idfile[0] idfile = idfile[0]
elif idfile.startswith('"') and idfile.endswith('"'): elif idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1] idfile = idfile[1:-1]
return {'hostname': sshconfig['hostname'], return {
'port': int(sshconfig['port']), 'hostname': sshconfig['hostname'],
'user': sshconfig['user'], 'port': int(sshconfig['port']),
'idfile': idfile} 'user': sshconfig['user'],
'idfile': idfile,
}
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise FDroidBuildVmException("Error getting ssh config") from e raise FDroidBuildVmException("Error getting ssh config") from e
@ -411,21 +443,27 @@ class LibvirtBuildVm(FDroidBuildVm):
# TODO use a libvirt storage pool to ensure the img file is readable # TODO use a libvirt storage pool to ensure the img file is readable
if not os.access(imagepath, os.R_OK): if not os.access(imagepath, os.R_OK):
logging.warning(_('Cannot read "{path}"!').format(path=imagepath)) logging.warning(_('Cannot read "{path}"!').format(path=imagepath))
_check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) _check_call(
['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']
)
shutil.copy2(imagepath, 'box.img') shutil.copy2(imagepath, 'box.img')
_check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img']) img_info_raw = _check_output(
['qemu-img', 'info', '--output=json', 'box.img']
)
img_info = json.loads(img_info_raw.decode('utf-8')) img_info = json.loads(img_info_raw.decode('utf-8'))
metadata = {"provider": "libvirt", metadata = {
"format": img_info['format'], "provider": "libvirt",
"virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)), "format": img_info['format'],
} "virtual_size": math.ceil(img_info['virtual-size'] / (1024.0 ** 3)),
}
logging.debug('preparing metadata.json for box %s', output) logging.debug('preparing metadata.json for box %s', output)
with open('metadata.json', 'w') as fp: with open('metadata.json', 'w') as fp:
fp.write(json.dumps(metadata)) fp.write(json.dumps(metadata))
logging.debug('preparing Vagrantfile for box %s', output) logging.debug('preparing Vagrantfile for box %s', output)
vagrantfile = textwrap.dedent("""\ vagrantfile = textwrap.dedent(
"""\
Vagrant.configure("2") do |config| Vagrant.configure("2") do |config|
config.ssh.username = "vagrant" config.ssh.username = "vagrant"
config.ssh.password = "vagrant" config.ssh.password = "vagrant"
@ -440,11 +478,18 @@ class LibvirtBuildVm(FDroidBuildVm):
libvirt.memory = {memory} libvirt.memory = {memory}
end end
end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])})) end""".format_map(
{
'memory': str(int(domainInfo[1] / 1024)),
'cpus': str(domainInfo[3]),
}
)
)
with open('Vagrantfile', 'w') as fp: with open('Vagrantfile', 'w') as fp:
fp.write(vagrantfile) fp.write(vagrantfile)
try: try:
import libarchive import libarchive
with libarchive.file_writer(output, 'gnutar', 'gzip') as tar: with libarchive.file_writer(output, 'gnutar', 'gzip') as tar:
logging.debug('adding files to box %s ...', output) logging.debug('adding files to box %s ...', output)
tar.add_files('metadata.json', 'Vagrantfile', 'box.img') tar.add_files('metadata.json', 'Vagrantfile', 'box.img')
@ -506,6 +551,7 @@ class LibvirtBuildVm(FDroidBuildVm):
def snapshot_exists(self, snapshot_name): def snapshot_exists(self, snapshot_name):
import libvirt import libvirt
try: try:
dom = self.conn.lookupByName(self.srvname) dom = self.conn.lookupByName(self.srvname)
return dom.snapshotLookupByName(snapshot_name) is not None return dom.snapshotLookupByName(snapshot_name) is not None
@ -515,6 +561,7 @@ class LibvirtBuildVm(FDroidBuildVm):
def snapshot_revert(self, snapshot_name): def snapshot_revert(self, snapshot_name):
logging.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name) logging.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name)
import libvirt import libvirt
try: try:
dom = self.conn.lookupByName(self.srvname) dom = self.conn.lookupByName(self.srvname)
snap = dom.snapshotLookupByName(snapshot_name) snap = dom.snapshotLookupByName(snapshot_name)