kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-20 13:50:12 +01:00
Code Activity

Merge branch 'partial-black-reformat' into 'master'

Browse Source 
run black to reformat code that does not have WIP merge requests

See merge request fdroid/fdroidserver!978
This commit is contained in:
Jochen Sprickerhof 2021-06-28 18:52:39 +00:00
commit e90212fb3a
9 changed files with 519 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fdroidserver/__init__.py
View File

@ -1,4 +1,3 @@
import gettext
import glob
import os
@ -8,7 +7,9 @@ import sys
# support running straight from git and standard installs
rootpaths = [
os.path.realpath(os.path.join(os.path.dirname(__file__), '..')),
os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share')),
os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', 'share')
),
os.path.join(sys.prefix, 'share'),
]

4
fdroidserver/gpgsign.py
View File

@ -59,7 +59,9 @@ def main():
signed = []
for output_dir in repodirs:
if not os.path.isdir(output_dir):
raise FDroidException(_("Missing output directory") + " '" + output_dir + "'")
raise FDroidException(
_("Missing output directory") + " '" + output_dir + "'"
)
# Process any apks that are waiting to be signed...
for f in sorted(glob.glob(os.path.join(output_dir, '*.*'))):

129
fdroidserver/init.py
View File

@ -38,6 +38,7 @@ options = None
def disable_in_config(key, value):
"""Write a key/value to the local config.yml, then comment it out."""
import yaml
with open('config.yml') as f:
data = f.read()
pattern = r'\n[\s#]*' + key + r':.*'
@ -54,16 +55,33 @@ def main():
# Parse command line...
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("-d", "--distinguished-name", default=None,
help=_("X.509 'Distinguished Name' used when generating keys"))
parser.add_argument("--keystore", default=None,
help=_("Path to the keystore for the repo signing key"))
parser.add_argument("--repo-keyalias", default=None,
help=_("Alias of the repo signing key in the keystore"))
parser.add_argument("--android-home", default=None,
help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)"))
parser.add_argument("--no-prompt", action="store_true", default=False,
help=_("Do not prompt for Android SDK path, just fail"))
parser.add_argument(
"-d",
"--distinguished-name",
default=None,
help=_("X.509 'Distinguished Name' used when generating keys"),
)
parser.add_argument(
"--keystore",
default=None,
help=_("Path to the keystore for the repo signing key"),
)
parser.add_argument(
"--repo-keyalias",
default=None,
help=_("Alias of the repo signing key in the keystore"),
)
parser.add_argument(
"--android-home",
default=None,
help=_("Path to the Android SDK (sometimes set in ANDROID_HOME)"),
)
parser.add_argument(
"--no-prompt",
action="store_true",
default=False,
help=_("Do not prompt for Android SDK path, just fail"),
)
options = parser.parse_args()
fdroiddir = os.getcwd()
@ -81,8 +99,9 @@ def main():
# and if the user leaves it blank, ignore and move on.
default_sdk_path = ''
if sys.platform == 'win32' or sys.platform == 'cygwin':
p = os.path.join(os.getenv('USERPROFILE'),
'AppData', 'Local', 'Android', 'android-sdk')
p = os.path.join(
os.getenv('USERPROFILE'), 'AppData', 'Local', 'Android', 'android-sdk'
)
elif sys.platform == 'darwin':
# on OSX, Homebrew is common and has an easy path to detect
p = '/usr/local/opt/android-sdk'
@ -96,10 +115,13 @@ def main():
test_config['sdk_path'] = default_sdk_path
if not common.test_sdk_exists(test_config):
del(test_config['sdk_path'])
del (test_config['sdk_path'])
while not options.no_prompt:
try:
s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path)
s = input(
_('Enter the path to the Android SDK (%s) here:\n> ')
% default_sdk_path
)
except KeyboardInterrupt:
print('')
sys.exit(1)
@ -112,8 +134,9 @@ def main():
default_sdk_path = ''
if test_config.get('sdk_path') and not common.test_sdk_exists(test_config):
raise FDroidException(_("Android SDK not found at {path}!")
.format(path=test_config['sdk_path']))
raise FDroidException(
_("Android SDK not found at {path}!").format(path=test_config['sdk_path'])
)
if not os.path.exists('config.yml') and not os.path.exists('config.py'):
# 'metadata' and 'tmp' are created in fdroid
@ -124,12 +147,14 @@ def main():
shutil.copyfile(example_config_yml, 'config.yml')
else:
from pkg_resources import get_distribution
versionstr = get_distribution('fdroidserver').version
if not versionstr:
versionstr = 'master'
with open('config.yml', 'w') as fp:
fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/'
+ versionstr + '/examples/config.yml\n')
fp.write('# see https://gitlab.com/fdroid/fdroidserver/blob/')
fp.write(versionstr)
fp.write('/examples/config.yml\n')
os.chmod('config.yml', 0o0600)
# If android_home is None, test_config['sdk_path'] will be used and
# "$ANDROID_HOME" may be used if the env var is set up correctly.
@ -138,7 +163,9 @@ def main():
if 'sdk_path' in test_config:
common.write_to_config(test_config, 'sdk_path', options.android_home)
else:
logging.warning('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
logging.warning(
'Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...'
)
logging.info('Try running `fdroid init` in an empty directory.')
raise FDroidException('Repository already exists.')
@ -162,8 +189,9 @@ def main():
else:
keystore = os.path.abspath(options.keystore)
if not os.path.exists(keystore):
logging.info('"' + keystore
+ '" does not exist, creating a new keystore there.')
logging.info(
'"' + keystore + '" does not exist, creating a new keystore there.'
)
common.write_to_config(test_config, 'keystore', keystore)
repo_keyalias = None
keydname = None
@ -174,12 +202,19 @@ def main():
keydname = options.distinguished_name
common.write_to_config(test_config, 'keydname', keydname)
if keystore == 'NONE': # we're using a smartcard
common.write_to_config(test_config, 'repo_keyalias', '1') # seems to be the default
common.write_to_config(
test_config, 'repo_keyalias', '1'
) # seems to be the default
disable_in_config('keypass', 'never used with smartcard')
common.write_to_config(test_config, 'smartcardoptions',
('-storetype PKCS11 '
+ '-providerClass sun.security.pkcs11.SunPKCS11 '
+ '-providerArg opensc-fdroid.cfg'))
common.write_to_config(
test_config,
'smartcardoptions',
(
'-storetype PKCS11 '
+ '-providerClass sun.security.pkcs11.SunPKCS11 '
+ '-providerArg opensc-fdroid.cfg'
),
)
# find opensc-pkcs11.so
if not os.path.exists('opensc-fdroid.cfg'):
if os.path.exists('/usr/lib/opensc-pkcs11.so'):
@ -187,29 +222,43 @@ def main():
elif os.path.exists('/usr/lib64/opensc-pkcs11.so'):
opensc_so = '/usr/lib64/opensc-pkcs11.so'
else:
files = glob.glob('/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so')
files = glob.glob(
'/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so'
)
if len(files) > 0:
opensc_so = files[0]
else:
opensc_so = '/usr/lib/opensc-pkcs11.so'
logging.warning('No OpenSC PKCS#11 module found, '
+ 'install OpenSC then edit "opensc-fdroid.cfg"!')
logging.warning(
'No OpenSC PKCS#11 module found, '
+ 'install OpenSC then edit "opensc-fdroid.cfg"!'
)
with open('opensc-fdroid.cfg', 'w') as f:
f.write('name = OpenSC\nlibrary = ')
f.write(opensc_so)
f.write('\n')
logging.info("Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml.")
logging.info("If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update "
"--create-key'.")
logging.info(
"Repo setup using a smartcard HSM. Please edit keystorepass and repo_keyalias in config.yml."
)
logging.info(
"If you want to generate a new repo signing key in the HSM you can do that with 'fdroid update "
"--create-key'."
)
elif os.path.exists(keystore):
to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname']
if repo_keyalias:
to_set.remove('repo_keyalias')
if keydname:
to_set.remove('keydname')
logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore)
+ '\n' + _('Now set these in config.yml:') + ' '
+ ', '.join(to_set) + '\n')
logging.warning(
'\n'
+ _('Using existing keystore "{path}"').format(path=keystore)
+ '\n'
+ _('Now set these in config.yml:')
+ ' '
+ ', '.join(to_set)
+ '\n'
)
else:
password = common.genpassword()
c = dict(test_config)
@ -229,11 +278,17 @@ def main():
msg += '\n ' + _('Keystore for signing key:\t') + keystore
if repo_keyalias is not None:
msg += '\n Alias for key in store:\t' + repo_keyalias
msg += '\n\n' + '''To complete the setup, add your APKs to "%s"
msg += '\n\n'
msg += (
_(
'''To complete the setup, add your APKs to "%s"
then run "fdroid update -c; fdroid update". You might also want to edit
"config.yml" to set the URL, repo name, and more. You should also set up
a signing key (a temporary one might have been automatically generated).
For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo
and https://f-droid.org/docs/Signing_Process''' % os.path.join(fdroiddir, 'repo')
and https://f-droid.org/docs/Signing_Process'''
)
% os.path.join(fdroiddir, 'repo')
)
logging.info(msg)

133
fdroidserver/mirror.py
View File

@ -33,8 +33,15 @@ def _run_wget(path, urls):
with open(urls_file, 'w') as fp:
for url in urls:
fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename
subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"',
'--input-file=' + urls_file])
subprocess.call(
[
'wget',
verbose,
'--continue',
'--user-agent="fdroid mirror"',
'--input-file=' + urls_file,
]
)
os.remove(urls_file)
@ -43,21 +50,47 @@ def main():
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("url", nargs='?',
help=_('Base URL to mirror, can include the index signing key '
+ 'using the query string: ?fingerprint='))
parser.add_argument("--all", action='store_true', default=False,
help=_("Mirror the full repo and archive, all file types."))
parser.add_argument("--archive", action='store_true', default=False,
help=_("Also mirror the full archive section"))
parser.add_argument("--build-logs", action='store_true', default=False,
help=_("Include the build logs in the mirror"))
parser.add_argument("--pgp-signatures", action='store_true', default=False,
help=_("Include the PGP signature .asc files in the mirror"))
parser.add_argument("--src-tarballs", action='store_true', default=False,
help=_("Include the source tarballs in the mirror"))
parser.add_argument("--output-dir", default=None,
help=_("The directory to write the mirror to"))
parser.add_argument(
"url",
nargs='?',
help=_(
'Base URL to mirror, can include the index signing key '
+ 'using the query string: ?fingerprint='
),
)
parser.add_argument(
"--all",
action='store_true',
default=False,
help=_("Mirror the full repo and archive, all file types."),
)
parser.add_argument(
"--archive",
action='store_true',
default=False,
help=_("Also mirror the full archive section"),
)
parser.add_argument(
"--build-logs",
action='store_true',
default=False,
help=_("Include the build logs in the mirror"),
)
parser.add_argument(
"--pgp-signatures",
action='store_true',
default=False,
help=_("Include the PGP signature .asc files in the mirror"),
)
parser.add_argument(
"--src-tarballs",
action='store_true',
default=False,
help=_("Include the source tarballs in the mirror"),
)
parser.add_argument(
"--output-dir", default=None, help=_("The directory to write the mirror to")
)
options = parser.parse_args()
if options.all:
@ -77,24 +110,31 @@ def main():
def _append_to_url_path(*args):
"""Append the list of path components to URL, keeping the rest the same."""
newpath = posixpath.join(path, *args)
return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment))
return urllib.parse.urlunparse(
(scheme, hostname, newpath, params, query, fragment)
)
if fingerprint:
config = common.read_config(options)
if not ('jarsigner' in config or 'apksigner' in config):
logging.error(_('Java JDK not found! Install in standard location or set java_paths!'))
logging.error(
_('Java JDK not found! Install in standard location or set java_paths!')
)
sys.exit(1)
def _get_index(section, etag=None):
url = _append_to_url_path(section)
data, etag = index.download_repo_index(url, etag=etag)
return data, etag, _append_to_url_path(section, 'index-v1.jar')
else:
def _get_index(section, etag=None):
import io
import json
import zipfile
from . import net
url = _append_to_url_path(section, 'index-v1.jar')
content, etag = net.http_get(url)
with zipfile.ZipFile(io.BytesIO(content)) as zip:
@ -107,21 +147,30 @@ def main():
ip = ipaddress.ip_address(hostname)
except ValueError:
pass
if hostname == 'f-droid.org' \
or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
'A full mirror of f-droid.org requires more than 200GB.'))
if hostname == 'f-droid.org' or (
ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]
):
print(
_(
'ERROR: this command should never be used to mirror f-droid.org!\n'
'A full mirror of f-droid.org requires more than 200GB.'
)
)
sys.exit(1)
path = path.rstrip('/')
if path.endswith('repo') or path.endswith('archive'):
logging.warning(_('Do not include "{path}" in URL!')
.format(path=path.split('/')[-1]))
logging.warning(
_('Do not include "{path}" in URL!').format(path=path.split('/')[-1])
)
elif not path.endswith('fdroid'):
logging.warning(_('{url} does not end with "fdroid", check the URL path!')
.format(url=options.url))
logging.warning(
_('{url} does not end with "fdroid", check the URL path!').format(
url=options.url
)
)
icondirs = ['icons', ]
icondirs = ['icons']
for density in update.screen_densities:
icondirs.append('icons-' + density)
@ -134,7 +183,7 @@ def main():
if options.archive:
sections = ('repo', 'archive')
else:
sections = ('repo', )
sections = ('repo',)
for section in sections:
sectiondir = os.path.join(basedir, section)
@ -152,23 +201,29 @@ def main():
for packageName, packageList in data['packages'].items():
for package in packageList:
to_fetch = []
keys = ['apkName', ]
keys = ['apkName']
if options.src_tarballs:
keys.append('srcname')
for k in keys:
if k in package:
to_fetch.append(package[k])
elif k == 'apkName':
logging.error(_('{appid} is missing {name}')
.format(appid=package['packageName'], name=k))
logging.error(
_('{appid} is missing {name}').format(
appid=package['packageName'], name=k
)
)
for f in to_fetch:
if not os.path.exists(f) \
or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
if not os.path.exists(f) or (
f.endswith('.apk') and os.path.getsize(f) != package['size']
):
urls.append(_append_to_url_path(section, f))
if options.pgp_signatures:
urls.append(_append_to_url_path(section, f + '.asc'))
if options.build_logs and f.endswith('.apk'):
urls.append(_append_to_url_path(section, f[:-4] + '.log.gz'))
urls.append(
_append_to_url_path(section, f[:-4] + '.log.gz')
)
_run_wget(sectiondir, urls)
@ -181,7 +236,7 @@ def main():
for k in update.GRAPHIC_NAMES:
f = d.get(k)
if f:
filepath_tuple = components + (f, )
filepath_tuple = components + (f,)
urls.append(_append_to_url_path(*filepath_tuple))
_run_wget(os.path.join(basedir, *components), urls)
for k in update.SCREENSHOT_DIRS:
@ -190,14 +245,16 @@ def main():
if filelist:
components = (section, app['packageName'], locale, k)
for f in filelist:
filepath_tuple = components + (f, )
filepath_tuple = components + (f,)
urls.append(_append_to_url_path(*filepath_tuple))
_run_wget(os.path.join(basedir, *components), urls)
urls = dict()
for app in data['apps']:
if 'icon' not in app:
logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
logging.error(
_('no "icon" in {appid}').format(appid=app['packageName'])
)
continue
icon = app['icon']
for icondir in icondirs:

196
fdroidserver/nightly.py
View File

@ -54,27 +54,72 @@ def _ssh_key_from_debug_keystore(keystore=KEYSTORE_FILE):
p12 = os.path.join(tmp_dir, '.keystore.p12')
_config = dict()
common.fill_config_defaults(_config)
subprocess.check_call([_config['keytool'], '-importkeystore',
'-srckeystore', keystore, '-srcalias', KEY_ALIAS,
'-srcstorepass', PASSWORD, '-srckeypass', PASSWORD,
'-destkeystore', p12, '-destalias', KEY_ALIAS,
'-deststorepass', PASSWORD, '-destkeypass', PASSWORD,
'-deststoretype', 'PKCS12'],
env={'LC_ALL': 'C.UTF-8'})
subprocess.check_call(['openssl', 'pkcs12', '-in', p12, '-out', key_pem,
'-passin', 'pass:' + PASSWORD, '-passout', 'pass:' + PASSWORD],
env={'LC_ALL': 'C.UTF-8'})
subprocess.check_call(['openssl', 'rsa', '-in', key_pem, '-out', privkey,
'-passin', 'pass:' + PASSWORD],
env={'LC_ALL': 'C.UTF-8'})
subprocess.check_call(
[
_config['keytool'],
'-importkeystore',
'-srckeystore',
keystore,
'-srcalias',
KEY_ALIAS,
'-srcstorepass',
PASSWORD,
'-srckeypass',
PASSWORD,
'-destkeystore',
p12,
'-destalias',
KEY_ALIAS,
'-deststorepass',
PASSWORD,
'-destkeypass',
PASSWORD,
'-deststoretype',
'PKCS12',
],
env={'LC_ALL': 'C.UTF-8'},
)
subprocess.check_call(
[
'openssl',
'pkcs12',
'-in',
p12,
'-out',
key_pem,
'-passin',
'pass:' + PASSWORD,
'-passout',
'pass:' + PASSWORD,
],
env={'LC_ALL': 'C.UTF-8'},
)
subprocess.check_call(
[
'openssl',
'rsa',
'-in',
key_pem,
'-out',
privkey,
'-passin',
'pass:' + PASSWORD,
],
env={'LC_ALL': 'C.UTF-8'},
)
os.remove(key_pem)
os.remove(p12)
os.chmod(privkey, 0o600) # os.umask() should cover this, but just in case
rsakey = paramiko.RSAKey.from_private_key_file(privkey)
fingerprint = base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest()).decode('ascii').rstrip('=')
ssh_private_key_file = os.path.join(tmp_dir, 'debug_keystore_'
+ fingerprint.replace('/', '_') + '_id_rsa')
fingerprint = (
base64.b64encode(hashlib.sha256(rsakey.asbytes()).digest())
.decode('ascii')
.rstrip('=')
)
ssh_private_key_file = os.path.join(
tmp_dir, 'debug_keystore_' + fingerprint.replace('/', '_') + '_id_rsa'
)
shutil.move(privkey, ssh_private_key_file)
pub = rsakey.get_name() + ' ' + rsakey.get_base64() + ' ' + ssh_private_key_file
@ -90,20 +135,46 @@ def main():
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("--keystore", default=KEYSTORE_FILE,
help=_("Specify which debug keystore file to use."))
parser.add_argument("--show-secret-var", action="store_true", default=False,
help=_("Print the secret variable to the terminal for easy copy/paste"))
parser.add_argument("--keep-private-keys", action="store_true", default=False,
help=_("Do not remove the private keys generated from the keystore"))
parser.add_argument("--no-deploy", action="store_true", default=False,
help=_("Do not deploy the new files to the repo"))
parser.add_argument("--file", default='app/build/outputs/apk/*.apk',
help=_('The file to be included in the repo (path or glob)'))
parser.add_argument("--no-checksum", action="store_true", default=False,
help=_("Don't use rsync checksums"))
parser.add_argument("--archive-older", type=int, default=20,
help=_("Set maximum releases in repo before older ones are archived"))
parser.add_argument(
"--keystore",
default=KEYSTORE_FILE,
help=_("Specify which debug keystore file to use."),
)
parser.add_argument(
"--show-secret-var",
action="store_true",
default=False,
help=_("Print the secret variable to the terminal for easy copy/paste"),
)
parser.add_argument(
"--keep-private-keys",
action="store_true",
default=False,
help=_("Do not remove the private keys generated from the keystore"),
)
parser.add_argument(
"--no-deploy",
action="store_true",
default=False,
help=_("Do not deploy the new files to the repo"),
)
parser.add_argument(
"--file",
default='app/build/outputs/apk/*.apk',
help=_('The file to be included in the repo (path or glob)'),
)
parser.add_argument(
"--no-checksum",
action="store_true",
default=False,
help=_("Don't use rsync checksums"),
)
parser.add_argument(
"--archive-older",
type=int,
default=20,
help=_("Set maximum releases in repo before older ones are archived"),
)
# TODO add --with-btlog
options = parser.parse_args()
@ -149,9 +220,11 @@ def main():
+ '\nhttps://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys')
git_user_name = repo_git_base
git_user_email = os.getenv('USER') + '@' + platform.node()
elif 'CIRCLE_REPOSITORY_URL' in os.environ \
and 'CIRCLE_PROJECT_USERNAME' in os.environ \
and 'CIRCLE_PROJECT_REPONAME' in os.environ:
elif (
'CIRCLE_REPOSITORY_URL' in os.environ
and 'CIRCLE_PROJECT_USERNAME' in os.environ
and 'CIRCLE_PROJECT_REPONAME' in os.environ
):
# we are in Circle CI
repo_git_base = (os.getenv('CIRCLE_PROJECT_USERNAME')
+ '/' + os.getenv('CIRCLE_PROJECT_REPONAME') + NIGHTLY)
@ -229,7 +302,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
config += "repo_description = 'Nightly builds from %s'\n" % git_user_email
config += "archive_name = '%s'\n" % (repo_git_base + ' archive')
config += "archive_url = '%s'\n" % (repo_base + '/archive')
config += "archive_description = 'Old nightly builds that have been archived.'\n"
config += (
"archive_description = 'Old nightly builds that have been archived.'\n"
)
config += "archive_older = %i\n" % options.archive_older
config += "servergitmirrors = '%s'\n" % servergitmirror
config += "keystore = '%s'\n" % KEYSTORE_FILE
@ -252,25 +327,38 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
for f in files:
if f.endswith('-debug.apk'):
apkfilename = os.path.join(root, f)
logging.debug(_('Stripping mystery signature from {apkfilename}')
.format(apkfilename=apkfilename))
logging.debug(
_('Stripping mystery signature from {apkfilename}').format(
apkfilename=apkfilename
)
)
destapk = os.path.join(repodir, os.path.basename(f))
os.chmod(apkfilename, 0o644)
logging.debug(_('Resigning {apkfilename} with provided debug.keystore')
.format(apkfilename=os.path.basename(apkfilename)))
logging.debug(
_(
'Resigning {apkfilename} with provided debug.keystore'
).format(apkfilename=os.path.basename(apkfilename))
)
common.apk_strip_v1_signatures(apkfilename, strip_manifest=True)
common.sign_apk(apkfilename, destapk, KEY_ALIAS)
if options.verbose:
logging.debug(_('attempting bare SSH connection to test deploy key:'))
try:
subprocess.check_call(['ssh', '-Tvi', ssh_private_key_file,
'-oIdentitiesOnly=yes', '-oStrictHostKeyChecking=no',
servergitmirror.split(':')[0]])
subprocess.check_call(
[
'ssh',
'-Tvi',
ssh_private_key_file,
'-oIdentitiesOnly=yes',
'-oStrictHostKeyChecking=no',
servergitmirror.split(':')[0],
]
)
except subprocess.CalledProcessError:
pass
app_url = clone_url[:-len(NIGHTLY)]
app_url = clone_url[: -len(NIGHTLY)]
template = dict()
template['AuthorName'] = clone_url.split('/')[4]
template['AuthorWebSite'] = '/'.join(clone_url.split('/')[:4])
@ -282,9 +370,13 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
with open('template.yml', 'w') as fp:
yaml.dump(template, fp)
subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'],
cwd=repo_basedir)
common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/')
subprocess.check_call(
['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'],
cwd=repo_basedir,
)
common.local_rsync(
options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
)
common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update app metadata")
@ -294,8 +386,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
cmd = ['fdroid', 'deploy', '--verbose', '--no-keep-git-mirror-archive']
subprocess.check_call(cmd, cwd=repo_basedir)
except subprocess.CalledProcessError:
logging.error(_('cannot publish update, did you set the deploy key?')
+ '\n' + deploy_key_url)
logging.error(
_('cannot publish update, did you set the deploy key?')
+ '\n'
+ deploy_key_url
)
sys.exit(1)
if not options.keep_private_keys:
@ -326,8 +421,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
if options.show_secret_var:
with open(options.keystore, 'rb') as fp:
debug_keystore = base64.standard_b64encode(fp.read()).decode('ascii')
print(_('\n{path} encoded for the DEBUG_KEYSTORE secret variable:')
.format(path=options.keystore))
print(
_('\n{path} encoded for the DEBUG_KEYSTORE secret variable:').format(
path=options.keystore
)
)
print(debug_keystore)
os.umask(umask)

110
fdroidserver/publish.py
View File

@ -76,11 +76,16 @@ def key_alias(appid):
def read_fingerprints_from_keystore():
"""Obtain a dictionary containing all singning-key fingerprints which are managed by F-Droid, grouped by appid."""
env_vars = {'LC_ALL': 'C.UTF-8',
'FDROID_KEY_STORE_PASS': config['keystorepass']}
cmd = [config['keytool'], '-list',
'-v', '-keystore', config['keystore'],
'-storepass:env', 'FDROID_KEY_STORE_PASS']
env_vars = {'LC_ALL': 'C.UTF-8', 'FDROID_KEY_STORE_PASS': config['keystorepass']}
cmd = [
config['keytool'],
'-list',
'-v',
'-keystore',
config['keystore'],
'-storepass:env',
'FDROID_KEY_STORE_PASS',
]
if config['keystore'] == 'NONE':
cmd += config['smartcardoptions']
p = FDroidPopen(cmd, envs=env_vars, output=False)
@ -116,8 +121,10 @@ def sign_sig_key_fingerprint_list(jar_file):
cmd += config['smartcardoptions']
else: # smardcards never use -keypass
cmd += '-keypass:env', 'FDROID_KEY_PASS'
env_vars = {'FDROID_KEY_STORE_PASS': config['keystorepass'],
'FDROID_KEY_PASS': config.get('keypass', "")}
env_vars = {
'FDROID_KEY_STORE_PASS': config['keystorepass'],
'FDROID_KEY_PASS': config.get('keypass', ""),
}
p = common.FDroidPopen(cmd, envs=env_vars)
if p.returncode != 0:
raise FDroidException("Failed to sign '{}'!".format(jar_file))
@ -201,24 +208,44 @@ def create_key_if_not_existing(keyalias):
"""
# See if we already have a key for this application, and
# if not generate one...
env_vars = {'LC_ALL': 'C.UTF-8',
'FDROID_KEY_STORE_PASS': config['keystorepass'],
'FDROID_KEY_PASS': config.get('keypass', "")}
cmd = [config['keytool'], '-list',
'-alias', keyalias, '-keystore', config['keystore'],
'-storepass:env', 'FDROID_KEY_STORE_PASS']
env_vars = {
'LC_ALL': 'C.UTF-8',
'FDROID_KEY_STORE_PASS': config['keystorepass'],
'FDROID_KEY_PASS': config.get('keypass', ""),
}
cmd = [
config['keytool'],
'-list',
'-alias',
keyalias,
'-keystore',
config['keystore'],
'-storepass:env',
'FDROID_KEY_STORE_PASS',
]
if config['keystore'] == 'NONE':
cmd += config['smartcardoptions']
p = FDroidPopen(cmd, envs=env_vars)
if p.returncode != 0:
logging.info("Key does not exist - generating...")
cmd = [config['keytool'], '-genkey',
'-keystore', config['keystore'],
'-alias', keyalias,
'-keyalg', 'RSA', '-keysize', '2048',
'-validity', '10000',
'-storepass:env', 'FDROID_KEY_STORE_PASS',
'-dname', config['keydname']]
cmd = [
config['keytool'],
'-genkey',
'-keystore',
config['keystore'],
'-alias',
keyalias,
'-keyalg',
'RSA',
'-keysize',
'2048',
'-validity',
'10000',
'-storepass:env',
'FDROID_KEY_STORE_PASS',
'-dname',
config['keydname'],
]
if config['keystore'] == 'NONE':
cmd += config['smartcardoptions']
else:
@ -235,11 +262,15 @@ def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] "
"[APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser = ArgumentParser(
usage="%(prog)s [options] " "[APPID[:VERCODE] [APPID[:VERCODE] ...]]"
)
common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*',
help=_("application ID with optional versionCode in the form APPID[:VERCODE]"))
parser.add_argument(
"appid",
nargs='*',
help=_("application ID with optional versionCode in the form APPID[:VERCODE]"),
)
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W
@ -247,7 +278,9 @@ def main():
config = common.read_config(options)
if not ('jarsigner' in config and 'keytool' in config):
logging.critical(_('Java JDK not found! Install in standard location or set java_paths!'))
logging.critical(
_('Java JDK not found! Install in standard location or set java_paths!')
)
sys.exit(1)
common.assert_config_keystore(config)
@ -279,16 +312,21 @@ def main():
allapps = metadata.read_metadata()
vercodes = common.read_pkg_args(options.appid, True)
common.get_metadata_files(vercodes) # only check appids
common.get_metadata_files(vercodes) # only check appids
signed_apks = dict()
generated_keys = dict()
allaliases = check_for_key_collisions(allapps)
logging.info(ngettext('{0} app, {1} key aliases',
'{0} apps, {1} key aliases', len(allapps)).format(len(allapps), len(allaliases)))
logging.info(
ngettext(
'{0} app, {1} key aliases', '{0} apps, {1} key aliases', len(allapps)
).format(len(allapps), len(allaliases))
)
# Process any APKs or ZIPs that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))
+ glob.glob(os.path.join(unsigned_dir, '*.zip'))):
for apkfile in sorted(
glob.glob(os.path.join(unsigned_dir, '*.apk'))
+ glob.glob(os.path.join(unsigned_dir, '*.zip'))
):
appid, vercode = common.publishednameinfo(apkfile)
apkfilename = os.path.basename(apkfile)
@ -302,8 +340,9 @@ def main():
# There ought to be valid metadata for this app, otherwise why are we
# trying to publish it?
if appid not in allapps:
logging.error("Unexpected {0} found in unsigned directory"
.format(apkfilename))
logging.error(
"Unexpected {0} found in unsigned directory".format(apkfilename)
)
sys.exit(1)
app = allapps[appid]
@ -359,7 +398,9 @@ def main():
signature_file, _ignored, manifest, v2_files = signingfiles
with open(signature_file, 'rb') as f:
devfp = common.signer_fingerprint_short(common.get_certificate(f.read()))
devfp = common.signer_fingerprint_short(
common.get_certificate(f.read())
)
devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp)
devsignedtmp = os.path.join(tmp_dir, devsigned)
@ -390,8 +431,7 @@ def main():
common.sign_apk(apkfile, signed_apk_path, keyalias)
if appid not in signed_apks:
signed_apks[appid] = []
signed_apks[appid].append({"keyalias": keyalias,
"filename": apkfile})
signed_apks[appid].append({"keyalias": keyalias, "filename": apkfile})
publish_source_tarball(apkfilename, unsigned_dir, output_dir)
logging.info('Published ' + apkfilename)

5
fdroidserver/signatures.py
View File

@ -89,8 +89,9 @@ def extract(options):
def main():
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("APK", nargs='*',
help=_("signed APK, either a file-path or HTTPS URL."))
parser.add_argument(
"APK", nargs='*', help=_("signed APK, either a file-path or HTTPS URL.")
)
parser.add_argument("--no-check-https", action="store_true", default=False)
options = parser.parse_args()

21
fdroidserver/signindex.py
View File

@ -41,10 +41,19 @@ def sign_jar(jar):
but then Android < 4.3 would not be able to verify it.
https://code.google.com/p/android/issues/detail?id=38321
"""
args = [config['jarsigner'], '-keystore', config['keystore'],
'-storepass:env', 'FDROID_KEY_STORE_PASS',
'-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
jar, config['repo_keyalias']]
args = [
config['jarsigner'],
'-keystore',
config['keystore'],
'-storepass:env',
'FDROID_KEY_STORE_PASS',
'-digestalg',
'SHA1',
'-sigalg',
'SHA1withRSA',
jar,
config['repo_keyalias'],
]
if config['keystore'] == 'NONE':
args += config['smartcardoptions']
else: # smardcards never use -keypass
@ -96,7 +105,9 @@ def main():
if 'jarsigner' not in config:
raise FDroidException(
_('Java jarsigner not found! Install in standard location or set java_paths!')
_(
'Java jarsigner not found! Install in standard location or set java_paths!'
)
)
repodirs = ['repo']

123
fdroidserver/vmtools.py
View File

@ -42,14 +42,18 @@ def get_clean_builder(serverdir):
vagrantfile = os.path.join(serverdir, 'Vagrantfile')
if not os.path.isfile(vagrantfile):
with open(vagrantfile, 'w') as f:
f.write(textwrap.dedent("""\
f.write(
textwrap.dedent(
"""\
# generated file, do not change.
Vagrant.configure("2") do |config|
config.vm.box = "buildserver"
config.vm.synced_folder ".", "/vagrant", disabled: true
end
"""))
"""
)
)
vm = get_build_vm(serverdir)
logging.info('destroying buildserver before build')
vm.destroy()
@ -119,25 +123,37 @@ def get_build_vm(srvdir, provider=None):
if kvm_installed and vbox_installed:
logging.debug('both kvm and vbox are installed.')
elif kvm_installed:
logging.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'')
logging.debug(
'libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\''
)
return LibvirtBuildVm(abssrvdir)
elif vbox_installed:
logging.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'')
logging.debug(
'virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\''
)
return VirtualboxBuildVm(abssrvdir)
else:
logging.debug('could not confirm that either virtualbox or kvm/libvirt are installed')
logging.debug(
'could not confirm that either virtualbox or kvm/libvirt are installed'
)
# try guessing provider from .../srvdir/.vagrant internals
vagrant_libvirt_path = os.path.join(abssrvdir, '.vagrant', 'machines',
'default', 'libvirt')
has_libvirt_machine = isdir(vagrant_libvirt_path) \
and len(os.listdir(vagrant_libvirt_path)) > 0
vagrant_virtualbox_path = os.path.join(abssrvdir, '.vagrant', 'machines',
'default', 'virtualbox')
has_vbox_machine = isdir(vagrant_virtualbox_path) \
and len(os.listdir(vagrant_virtualbox_path)) > 0
vagrant_libvirt_path = os.path.join(
abssrvdir, '.vagrant', 'machines', 'default', 'libvirt'
)
has_libvirt_machine = (
isdir(vagrant_libvirt_path) and len(os.listdir(vagrant_libvirt_path)) > 0
)
vagrant_virtualbox_path = os.path.join(
abssrvdir, '.vagrant', 'machines', 'default', 'virtualbox'
)
has_vbox_machine = (
isdir(vagrant_virtualbox_path) and len(os.listdir(vagrant_virtualbox_path)) > 0
)
if has_libvirt_machine and has_vbox_machine:
logging.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
logging.info(
'build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\''
)
return VirtualboxBuildVm(abssrvdir)
elif has_libvirt_machine:
logging.debug('build vm provider lookup found \'libvirt\'')
@ -149,12 +165,15 @@ def get_build_vm(srvdir, provider=None):
# try guessing provider from available buildserver boxes
available_boxes = []
import vagrant
boxes = vagrant.Vagrant().box_list()
for box in boxes:
if box.name == "buildserver":
available_boxes.append(box.provider)
if "libvirt" in available_boxes and "virtualbox" in available_boxes:
logging.info('basebox lookup found virtualbox and libvirt boxes, defaulting to \'virtualbox\'')
logging.info(
'basebox lookup found virtualbox and libvirt boxes, defaulting to \'virtualbox\''
)
return VirtualboxBuildVm(abssrvdir)
elif "libvirt" in available_boxes:
logging.info('\'libvirt\' buildserver box available, using that')
@ -171,7 +190,7 @@ class FDroidBuildVmException(FDroidException):
pass
class FDroidBuildVm():
class FDroidBuildVm:
"""Abstract base class for working with FDroids build-servers.
Use the factory method `fdroidserver.vmtools.get_build_vm()` for
@ -188,11 +207,18 @@ class FDroidBuildVm():
self.vgrntfile = os.path.join(srvdir, 'Vagrantfile')
self.srvuuid = self._vagrant_fetch_uuid()
if not isdir(srvdir):
raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
raise FDroidBuildVmException(
"Can not init vagrant, directory %s not present" % (srvdir)
)
if not isfile(self.vgrntfile):
raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
raise FDroidBuildVmException(
"Can not init vagrant, '%s' not present" % (self.vgrntfile)
)
import vagrant
self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
self.vgrnt = vagrant.Vagrant(
root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm
)
def up(self, provision=True):
global lock
@ -302,7 +328,9 @@ class FDroidBuildVm():
"""
boxfile = abspath(boxfile)
if not isfile(boxfile):
raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile)
raise FDroidBuildVmException(
'supplied boxfile \'%s\' does not exist', boxfile
)
self.vgrnt.box_add(boxname, abspath(boxfile), force=force)
def box_remove(self, boxname):
@ -310,11 +338,13 @@ class FDroidBuildVm():
_check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
except subprocess.CalledProcessError as e:
logging.debug('tried removing box %s, but is did not exist: %s', boxname, e)
boxpath = os.path.join(expanduser('~'), '.vagrant',
self._vagrant_file_name(boxname))
boxpath = os.path.join(
expanduser('~'), '.vagrant', self._vagrant_file_name(boxname)
)
if isdir(boxpath):
logging.info("attempting to remove box '%s' by deleting: %s",
boxname, boxpath)
logging.info(
"attempting to remove box '%s' by deleting: %s", boxname, boxpath
)
shutil.rmtree(boxpath)
def sshinfo(self):
@ -325,11 +355,11 @@ class FDroidBuildVm():
A dictionary containing 'hostname', 'port', 'user' and 'idfile'
"""
import paramiko
try:
sshconfig_path = os.path.join(self.srvdir, 'sshconfig')
with open(sshconfig_path, 'wb') as fp:
fp.write(_check_output(['vagrant', 'ssh-config'],
cwd=self.srvdir))
fp.write(_check_output(['vagrant', 'ssh-config'], cwd=self.srvdir))
vagranthost = 'default' # Host in ssh config file
sshconfig = paramiko.SSHConfig()
with open(sshconfig_path, 'r') as f:
@ -340,10 +370,12 @@ class FDroidBuildVm():
idfile = idfile[0]
elif idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
return {'hostname': sshconfig['hostname'],
'port': int(sshconfig['port']),
'user': sshconfig['user'],
'idfile': idfile}
return {
'hostname': sshconfig['hostname'],
'port': int(sshconfig['port']),
'user': sshconfig['user'],
'idfile': idfile,
}
except subprocess.CalledProcessError as e:
raise FDroidBuildVmException("Error getting ssh config") from e
@ -411,21 +443,27 @@ class LibvirtBuildVm(FDroidBuildVm):
# TODO use a libvirt storage pool to ensure the img file is readable
if not os.access(imagepath, os.R_OK):
logging.warning(_('Cannot read "{path}"!').format(path=imagepath))
_check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
_check_call(
['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']
)
shutil.copy2(imagepath, 'box.img')
_check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
img_info_raw = _check_output(
['qemu-img', 'info', '--output=json', 'box.img']
)
img_info = json.loads(img_info_raw.decode('utf-8'))
metadata = {"provider": "libvirt",
"format": img_info['format'],
"virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
}
metadata = {
"provider": "libvirt",
"format": img_info['format'],
"virtual_size": math.ceil(img_info['virtual-size'] / (1024.0 ** 3)),
}
logging.debug('preparing metadata.json for box %s', output)
with open('metadata.json', 'w') as fp:
fp.write(json.dumps(metadata))
logging.debug('preparing Vagrantfile for box %s', output)
vagrantfile = textwrap.dedent("""\
vagrantfile = textwrap.dedent(
"""\
Vagrant.configure("2") do |config|
config.ssh.username = "vagrant"
config.ssh.password = "vagrant"
@ -440,11 +478,18 @@ class LibvirtBuildVm(FDroidBuildVm):
libvirt.memory = {memory}
end
end""".format_map({'memory': str(int(domainInfo[1] / 1024)), 'cpus': str(domainInfo[3])}))
end""".format_map(
{
'memory': str(int(domainInfo[1] / 1024)),
'cpus': str(domainInfo[3]),
}
)
)
with open('Vagrantfile', 'w') as fp:
fp.write(vagrantfile)
try:
import libarchive
with libarchive.file_writer(output, 'gnutar', 'gzip') as tar:
logging.debug('adding files to box %s ...', output)
tar.add_files('metadata.json', 'Vagrantfile', 'box.img')
@ -506,6 +551,7 @@ class LibvirtBuildVm(FDroidBuildVm):
def snapshot_exists(self, snapshot_name):
import libvirt
try:
dom = self.conn.lookupByName(self.srvname)
return dom.snapshotLookupByName(snapshot_name) is not None
@ -515,6 +561,7 @@ class LibvirtBuildVm(FDroidBuildVm):
def snapshot_revert(self, snapshot_name):
logging.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name)
import libvirt
try:
dom = self.conn.lookupByName(self.srvname)
snap = dom.snapshotLookupByName(snapshot_name)