diff --git a/fdroidserver/common.py b/fdroidserver/common.py index f670282e..c1111939 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -34,7 +34,6 @@ import logging import hashlib import socket import base64 -import zipfile import xml.etree.ElementTree as XMLElementTree from binascii import hexlify @@ -390,47 +389,6 @@ def write_password_file(pwtype, password=None): config[pwtype + 'file'] = filename -def signjar(jar): - ''' - sign a JAR file with Java's jarsigner. - - This does use old hashing algorithms, i.e. SHA1, but that's not - broken yet for file verification. This could be set to SHA256, - but then Android < 4.3 would not be able to verify it. - https://code.google.com/p/android/issues/detail?id=38321 - ''' - args = [config['jarsigner'], '-keystore', config['keystore'], - '-storepass:file', config['keystorepassfile'], - '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA', - jar, config['repo_keyalias']] - if config['keystore'] == 'NONE': - args += config['smartcardoptions'] - else: # smardcards never use -keypass - args += ['-keypass:file', config['keypassfile']] - p = FDroidPopen(args) - if p.returncode != 0: - logging.critical("Failed to sign %s!" % jar) - sys.exit(1) - - -def sign_index_v1(repodir, json_name): - """ - sign index-v1.json to make index-v1.jar - - This is a bit different than index.jar: instead of their being index.xml - and index_unsigned.jar, the presense of index-v1.json means that there is - unsigned data. That file is then stuck into a jar and signed by the - signing process. index-v1.json is never published to the repo. It is - included in the binary transparency log, if that is enabled. - """ - name, ext = get_extension(json_name) - index_file = os.path.join(repodir, json_name) - jar_file = os.path.join(repodir, name + '.jar') - with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar: - jar.write(index_file, json_name) - signjar(jar_file) - - def get_local_metadata_files(): '''get any metadata files local to an app's source repo diff --git a/fdroidserver/index.py b/fdroidserver/index.py new file mode 100644 index 00000000..d725aa3a --- /dev/null +++ b/fdroidserver/index.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python3 +# +# update.py - part of the FDroid server tools +# Copyright (C) 2017, Torsten Grote +# Copyright (C) 2016, Blue Jay Wireless +# Copyright (C) 2014-2016, Hans-Christoph Steiner +# Copyright (C) 2010-2015, Ciaran Gultnieks +# Copyright (C) 2013-2014, Daniel Martí +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import collections +import copy +import json +import logging +import os +import re +import shutil +import sys +import urllib.parse +from binascii import hexlify, unhexlify +from datetime import datetime +from xml.dom.minidom import Document + +from fdroidserver import metadata, signindex, common +from fdroidserver.common import FDroidPopen, FDroidPopenBytes +from fdroidserver.metadata import MetaDataException + + +def make(apps, sortedids, apks, repodir, archive): + """Generate the repo index files. + + This requires properly initialized options and config objects. + + :param apps: fully populated apps list + :param sortedids: app package IDs, sorted + :param apks: full populated apks list + :param repodir: the repo directory + :param archive: True if this is the archive repo, False if it's the + main one. + """ + from fdroidserver.update import METADATA_VERSION + + def _resolve_description_link(appid): + if appid in apps: + return "fdroid.app:" + appid, apps[appid].Name + raise MetaDataException("Cannot resolve app id " + appid) + + nosigningkey = False + if not common.options.nosign: + if 'repo_keyalias' not in common.config: + nosigningkey = True + logging.critical("'repo_keyalias' not found in config.py!") + if 'keystore' not in common.config: + nosigningkey = True + logging.critical("'keystore' not found in config.py!") + if 'keystorepass' not in common.config and 'keystorepassfile' not in common.config: + nosigningkey = True + logging.critical("'keystorepass' not found in config.py!") + if 'keypass' not in common.config and 'keypassfile' not in common.config: + nosigningkey = True + logging.critical("'keypass' not found in config.py!") + if not os.path.exists(common.config['keystore']): + nosigningkey = True + logging.critical("'" + common.config['keystore'] + "' does not exist!") + if nosigningkey: + logging.warning("`fdroid update` requires a signing key, you can create one using:") + logging.warning("\tfdroid update --create-key") + sys.exit(1) + + repodict = collections.OrderedDict() + repodict['timestamp'] = datetime.utcnow() + repodict['version'] = METADATA_VERSION + + if common.config['repo_maxage'] != 0: + repodict['maxage'] = common.config['repo_maxage'] + + if archive: + repodict['name'] = common.config['archive_name'] + repodict['icon'] = os.path.basename(common.config['archive_icon']) + repodict['address'] = common.config['archive_url'] + repodict['description'] = common.config['archive_description'] + urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['archive_url']).path) + else: + repodict['name'] = common.config['repo_name'] + repodict['icon'] = os.path.basename(common.config['repo_icon']) + repodict['address'] = common.config['repo_url'] + repodict['description'] = common.config['repo_description'] + urlbasepath = os.path.basename(urllib.parse.urlparse(common.config['repo_url']).path) + + mirrorcheckfailed = False + mirrors = [] + for mirror in sorted(common.config.get('mirrors', [])): + base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) + if common.config.get('nonstandardwebroot') is not True and base != 'fdroid': + logging.error("mirror '" + mirror + "' does not end with 'fdroid'!") + mirrorcheckfailed = True + # must end with / or urljoin strips a whole path segment + if mirror.endswith('/'): + mirrors.append(urllib.parse.urljoin(mirror, urlbasepath)) + else: + mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath)) + for mirror in common.config.get('servergitmirrors', []): + mirror = get_raw_mirror(mirror) + if mirror is not None: + mirrors.append(mirror + '/') + if mirrorcheckfailed: + sys.exit(1) + if mirrors: + repodict['mirrors'] = mirrors + + appsWithPackages = collections.OrderedDict() + for packageName in sortedids: + app = apps[packageName] + if app['Disabled']: + continue + + # only include apps with packages + for apk in apks: + if apk['packageName'] == packageName: + newapp = copy.copy(app) # update wiki needs unmodified description + newapp['Description'] = metadata.description_html(app['Description'], + _resolve_description_link) + appsWithPackages[packageName] = newapp + break + + requestsdict = dict() + for command in ('install', 'uninstall'): + packageNames = [] + key = command + '_list' + if key in common.config: + if isinstance(common.config[key], str): + packageNames = [common.config[key]] + elif all(isinstance(item, str) for item in common.config[key]): + packageNames = common.config[key] + else: + raise TypeError('only accepts strings, lists, and tuples') + requestsdict[command] = packageNames + + make_v0(appsWithPackages, apks, repodir, repodict, requestsdict) + make_v1(appsWithPackages, apks, repodir, repodict, requestsdict) + + +def make_v1(apps, packages, repodir, repodict, requestsdict): + + def _index_encoder_default(obj): + if isinstance(obj, set): + return list(obj) + if isinstance(obj, datetime): + return int(obj.timestamp() * 1000) # Java expects milliseconds + raise TypeError(repr(obj) + " is not JSON serializable") + + output = collections.OrderedDict() + output['repo'] = repodict + output['requests'] = requestsdict + + appslist = [] + output['apps'] = appslist + for appid, appdict in apps.items(): + d = collections.OrderedDict() + appslist.append(d) + for k, v in sorted(appdict.items()): + if not v: + continue + if k in ('builds', 'comments', 'metadatapath', + 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes', + 'Provides', 'Repo', 'RepoType', 'RequiresRoot', + 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode', + 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'): + continue + + # name things after the App class fields in fdroidclient + if k == 'id': + k = 'packageName' + elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name + k = 'suggestedVersionCode' + elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name + k = 'suggestedVersionName' + elif k == 'AutoName': + if 'Name' not in apps[appid]: + d['name'] = v + continue + else: + k = k[:1].lower() + k[1:] + d[k] = v + + output_packages = dict() + output['packages'] = output_packages + for package in packages: + packageName = package['packageName'] + if packageName in output_packages: + packagelist = output_packages[packageName] + else: + packagelist = [] + output_packages[packageName] = packagelist + d = collections.OrderedDict() + packagelist.append(d) + for k, v in sorted(package.items()): + if not v: + continue + if k in ('icon', 'icons', 'icons_src', 'name', ): + continue + d[k] = v + + json_name = 'index-v1.json' + index_file = os.path.join(repodir, json_name) + with open(index_file, 'w') as fp: + json.dump(output, fp, default=_index_encoder_default) + + if common.options.nosign: + logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!') + else: + signindex.config = common.config + signindex.sign_index_v1(repodir, json_name) + + +def make_v0(apps, apks, repodir, repodict, requestsdict): + """ + aka index.jar aka index.xml + """ + + doc = Document() + + def addElement(name, value, doc, parent): + el = doc.createElement(name) + el.appendChild(doc.createTextNode(value)) + parent.appendChild(el) + + def addElementNonEmpty(name, value, doc, parent): + if not value: + return + addElement(name, value, doc, parent) + + def addElementIfInApk(name, apk, key, doc, parent): + if key not in apk: + return + value = str(apk[key]) + addElement(name, value, doc, parent) + + def addElementCDATA(name, value, doc, parent): + el = doc.createElement(name) + el.appendChild(doc.createCDATASection(value)) + parent.appendChild(el) + + root = doc.createElement("fdroid") + doc.appendChild(root) + + repoel = doc.createElement("repo") + + repoel.setAttribute("name", repodict['name']) + if 'maxage' in repodict: + repoel.setAttribute("maxage", str(repodict['maxage'])) + repoel.setAttribute("icon", os.path.basename(repodict['icon'])) + repoel.setAttribute("url", repodict['address']) + addElement('description', repodict['description'], doc, repoel) + for mirror in repodict.get('mirrors', []): + addElement('mirror', mirror, doc, repoel) + + repoel.setAttribute("version", str(repodict['version'])) + repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp()) + + pubkey, repo_pubkey_fingerprint = extract_pubkey() + repoel.setAttribute("pubkey", pubkey.decode('utf-8')) + root.appendChild(repoel) + + for command in ('install', 'uninstall'): + for packageName in requestsdict[command]: + element = doc.createElement(command) + root.appendChild(element) + element.setAttribute('packageName', packageName) + + for appid, appdict in apps.items(): + app = metadata.App(appdict) + + if app.Disabled is not None: + continue + + # Get a list of the apks for this app... + apklist = [] + for apk in apks: + if apk['packageName'] == appid: + apklist.append(apk) + + if len(apklist) == 0: + continue + + apel = doc.createElement("application") + apel.setAttribute("id", app.id) + root.appendChild(apel) + + addElement('id', app.id, doc, apel) + if app.added: + addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel) + if app.lastUpdated: + addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel) + addElement('name', app.Name, doc, apel) + addElement('summary', app.Summary, doc, apel) + if app.icon: + addElement('icon', app.icon, doc, apel) + + if app.get('Description'): + description = app.Description + else: + description = '

No description available

' + addElement('desc', description, doc, apel) + addElement('license', app.License, doc, apel) + if app.Categories: + addElement('categories', ','.join(app.Categories), doc, apel) + # We put the first (primary) category in LAST, which will have + # the desired effect of making clients that only understand one + # category see that one. + addElement('category', app.Categories[0], doc, apel) + addElement('web', app.WebSite, doc, apel) + addElement('source', app.SourceCode, doc, apel) + addElement('tracker', app.IssueTracker, doc, apel) + addElementNonEmpty('changelog', app.Changelog, doc, apel) + addElementNonEmpty('author', app.AuthorName, doc, apel) + addElementNonEmpty('email', app.AuthorEmail, doc, apel) + addElementNonEmpty('donate', app.Donate, doc, apel) + addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel) + addElementNonEmpty('litecoin', app.Litecoin, doc, apel) + addElementNonEmpty('flattr', app.FlattrID, doc, apel) + + # These elements actually refer to the current version (i.e. which + # one is recommended. They are historically mis-named, and need + # changing, but stay like this for now to support existing clients. + addElement('marketversion', app.CurrentVersion, doc, apel) + addElement('marketvercode', app.CurrentVersionCode, doc, apel) + + if app.Provides: + pv = app.Provides.split(',') + addElementNonEmpty('provides', ','.join(pv), doc, apel) + if app.RequiresRoot: + addElement('requirements', 'root', doc, apel) + + # Sort the apk list into version order, just so the web site + # doesn't have to do any work by default... + apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True) + + if 'antiFeatures' in apklist[0]: + app.AntiFeatures.extend(apklist[0]['antiFeatures']) + if app.AntiFeatures: + addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel) + + # Check for duplicates - they will make the client unhappy... + for i in range(len(apklist) - 1): + if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']: + logging.critical("duplicate versions: '%s' - '%s'" % ( + apklist[i]['apkName'], apklist[i + 1]['apkName'])) + sys.exit(1) + + current_version_code = 0 + current_version_file = None + for apk in apklist: + file_extension = common.get_file_extension(apk['apkName']) + # find the APK for the "Current Version" + if current_version_code < apk['versionCode']: + current_version_code = apk['versionCode'] + if current_version_code < int(app.CurrentVersionCode): + current_version_file = apk['apkName'] + + apkel = doc.createElement("package") + apel.appendChild(apkel) + addElement('version', apk['versionName'], doc, apkel) + addElement('versioncode', str(apk['versionCode']), doc, apkel) + addElement('apkname', apk['apkName'], doc, apkel) + addElementIfInApk('srcname', apk, 'srcname', doc, apkel) + + hashel = doc.createElement("hash") + hashel.setAttribute('type', 'sha256') + hashel.appendChild(doc.createTextNode(apk['hash'])) + apkel.appendChild(hashel) + + addElement('size', str(apk['size']), doc, apkel) + addElementIfInApk('sdkver', apk, + 'minSdkVersion', doc, apkel) + addElementIfInApk('targetSdkVersion', apk, + 'targetSdkVersion', doc, apkel) + addElementIfInApk('maxsdkver', apk, + 'maxSdkVersion', doc, apkel) + addElementIfInApk('obbMainFile', apk, + 'obbMainFile', doc, apkel) + addElementIfInApk('obbMainFileSha256', apk, + 'obbMainFileSha256', doc, apkel) + addElementIfInApk('obbPatchFile', apk, + 'obbPatchFile', doc, apkel) + addElementIfInApk('obbPatchFileSha256', apk, + 'obbPatchFileSha256', doc, apkel) + if 'added' in apk: + addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel) + + if file_extension == 'apk': # sig is required for APKs, but only APKs + addElement('sig', apk['sig'], doc, apkel) + + old_permissions = set() + sorted_permissions = sorted(apk['uses-permission']) + for perm in sorted_permissions: + perm_name = perm.name + if perm_name.startswith("android.permission."): + perm_name = perm_name[19:] + old_permissions.add(perm_name) + addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel) + + for permission in sorted_permissions: + permel = doc.createElement('uses-permission') + permel.setAttribute('name', permission.name) + if permission.maxSdkVersion is not None: + permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion) + apkel.appendChild(permel) + for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']): + permel = doc.createElement('uses-permission-sdk-23') + permel.setAttribute('name', permission_sdk_23.name) + if permission_sdk_23.maxSdkVersion is not None: + permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion) + apkel.appendChild(permel) + if 'nativecode' in apk: + addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel) + addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel) + + if current_version_file is not None \ + and common.config['make_current_version_link'] \ + and repodir == 'repo': # only create these + namefield = common.config['current_version_name_source'] + sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield)) + apklinkname = sanitized_name + '.apk' + current_version_path = os.path.join(repodir, current_version_file) + if os.path.islink(apklinkname): + os.remove(apklinkname) + os.symlink(current_version_path, apklinkname) + # also symlink gpg signature, if it exists + for extension in ('.asc', '.sig'): + sigfile_path = current_version_path + extension + if os.path.exists(sigfile_path): + siglinkname = apklinkname + extension + if os.path.islink(siglinkname): + os.remove(siglinkname) + os.symlink(sigfile_path, siglinkname) + + if common.options.pretty: + output = doc.toprettyxml(encoding='utf-8') + else: + output = doc.toxml(encoding='utf-8') + + with open(os.path.join(repodir, 'index.xml'), 'wb') as f: + f.write(output) + + if 'repo_keyalias' in common.config: + + if common.options.nosign: + logging.info("Creating unsigned index in preparation for signing") + else: + logging.info("Creating signed index with this key (SHA256):") + logging.info("%s" % repo_pubkey_fingerprint) + + # Create a jar of the index... + jar_output = 'index_unsigned.jar' if common.options.nosign else 'index.jar' + p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir) + if p.returncode != 0: + logging.critical("Failed to create {0}".format(jar_output)) + sys.exit(1) + + # Sign the index... + signed = os.path.join(repodir, 'index.jar') + if common.options.nosign: + # Remove old signed index if not signing + if os.path.exists(signed): + os.remove(signed) + else: + signindex.config = common.config + signindex.sign_jar(signed) + + # Copy the repo icon into the repo directory... + icon_dir = os.path.join(repodir, 'icons') + iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon'])) + shutil.copyfile(common.config['repo_icon'], iconfilename) + + +def extract_pubkey(): + """ + Extracts and returns the repository's public key from the keystore. + :return: public key in hex, repository fingerprint + """ + if 'repo_pubkey' in common.config: + pubkey = unhexlify(common.config['repo_pubkey']) + else: + p = FDroidPopenBytes([common.config['keytool'], '-exportcert', + '-alias', common.config['repo_keyalias'], + '-keystore', common.config['keystore'], + '-storepass:file', common.config['keystorepassfile']] + + common.config['smartcardoptions'], + output=False, stderr_to_stdout=False) + if p.returncode != 0 or len(p.output) < 20: + msg = "Failed to get repo pubkey!" + if common.config['keystore'] == 'NONE': + msg += ' Is your crypto smartcard plugged in?' + logging.critical(msg) + sys.exit(1) + pubkey = p.output + repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey) + return hexlify(pubkey), repo_pubkey_fingerprint + + +# Get raw URL from git service for mirroring +def get_raw_mirror(url): + # Divide urls in parts + url = url.split("/") + + # Get the hostname + hostname = url[2] + + # fdroidserver will use always 'master' branch for git-mirroring + branch = "master" + folder = "fdroid" + + if hostname == "github.com": + # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid" + url[2] = "raw.githubusercontent.com" + url.extend([branch, folder]) + elif hostname == "gitlab.com": + # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid" + url.extend(["raw", branch, folder]) + else: + return None + + url = "/".join(url) + return url diff --git a/fdroidserver/signindex.py b/fdroidserver/signindex.py index cbc19aa0..4f0ac678 100644 --- a/fdroidserver/signindex.py +++ b/fdroidserver/signindex.py @@ -18,6 +18,7 @@ import sys import os +import zipfile from argparse import ArgumentParser import logging @@ -27,6 +28,49 @@ config = None options = None +def sign_jar(jar): + """ + Sign a JAR file with Java's jarsigner. + + This method requires a properly initialized config object. + + This does use old hashing algorithms, i.e. SHA1, but that's not + broken yet for file verification. This could be set to SHA256, + but then Android < 4.3 would not be able to verify it. + https://code.google.com/p/android/issues/detail?id=38321 + """ + args = [config['jarsigner'], '-keystore', config['keystore'], + '-storepass:file', config['keystorepassfile'], + '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA', + jar, config['repo_keyalias']] + if config['keystore'] == 'NONE': + args += config['smartcardoptions'] + else: # smardcards never use -keypass + args += ['-keypass:file', config['keypassfile']] + p = common.FDroidPopen(args) + if p.returncode != 0: + logging.critical("Failed to sign %s!" % jar) + sys.exit(1) + + +def sign_index_v1(repodir, json_name): + """ + Sign index-v1.json to make index-v1.jar + + This is a bit different than index.jar: instead of their being index.xml + and index_unsigned.jar, the presence of index-v1.json means that there is + unsigned data. That file is then stuck into a jar and signed by the + signing process. index-v1.json is never published to the repo. It is + included in the binary transparency log, if that is enabled. + """ + name, ext = common.get_extension(json_name) + index_file = os.path.join(repodir, json_name) + jar_file = os.path.join(repodir, name + '.jar') + with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar: + jar.write(index_file, json_name) + sign_jar(jar_file) + + def main(): global config, options @@ -54,7 +98,7 @@ def main(): unsigned = os.path.join(output_dir, 'index_unsigned.jar') if os.path.exists(unsigned): - common.signjar(unsigned) + sign_jar(unsigned) os.rename(unsigned, os.path.join(output_dir, 'index.jar')) logging.info('Signed index in ' + output_dir) signed += 1 @@ -62,7 +106,7 @@ def main(): json_name = 'index-v1.json' index_file = os.path.join(output_dir, json_name) if os.path.exists(index_file): - common.sign_index_v1(output_dir, json_name) + sign_index_v1(output_dir, json_name) os.remove(index_file) logging.info('Signed ' + index_file) signed += 1 diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 096323dd..ee8accc2 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -19,7 +19,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import copy import sys import os import shutil @@ -31,24 +30,22 @@ import zipfile import hashlib import pickle import platform -import urllib.parse from datetime import datetime, timedelta -from xml.dom.minidom import Document from argparse import ArgumentParser import collections from pyasn1.error import PyAsn1Error from pyasn1.codec.der import decoder, encoder from pyasn1_modules import rfc2315 -from binascii import hexlify, unhexlify +from binascii import hexlify from PIL import Image import logging from . import common +from . import index from . import metadata -from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen -from .metadata import MetaDataException +from .common import FDroidPopen, SdkToolsPopen METADATA_VERSION = 18 @@ -1104,31 +1101,6 @@ def scan_apks(apkcache, repodir, knownapks, use_date_from_apk=False): return apks, cachechanged -repo_pubkey_fingerprint = None - - -def extract_pubkey(): - global repo_pubkey_fingerprint - if 'repo_pubkey' in config: - pubkey = unhexlify(config['repo_pubkey']) - else: - p = FDroidPopenBytes([config['keytool'], '-exportcert', - '-alias', config['repo_keyalias'], - '-keystore', config['keystore'], - '-storepass:file', config['keystorepassfile']] - + config['smartcardoptions'], - output=False, stderr_to_stdout=False) - if p.returncode != 0 or len(p.output) < 20: - msg = "Failed to get repo pubkey!" - if config['keystore'] == 'NONE': - msg += ' Is your crypto smartcard plugged in?' - logging.critical(msg) - sys.exit(1) - pubkey = p.output - repo_pubkey_fingerprint = common.get_cert_fingerprint(pubkey) - return hexlify(pubkey) - - def apply_info_from_latest_apk(apps, apks): """ Some information from the apks needs to be applied up to the application level. @@ -1168,472 +1140,6 @@ def apply_info_from_latest_apk(apps, apks): app.CurrentVersionCode = str(bestver) -# Get raw URL from git service for mirroring -def get_raw_mirror(url): - # Divide urls in parts - url = url.split("/") - - # Get the hostname - hostname = url[2] - - # fdroidserver will use always 'master' branch for git-mirroring - branch = "master" - folder = "fdroid" - - if hostname == "github.com": - # Github like RAW url "https://raw.githubusercontent.com/user/repo/master/fdroid" - url[2] = "raw.githubusercontent.com" - url.extend([branch, folder]) - elif hostname == "gitlab.com": - # Gitlab like RAW url "https://gitlab.com/user/repo/raw/master/fdroid" - url.extend(["raw", branch, folder]) - else: - return None - - url = "/".join(url) - return url - - -def make_index(apps, sortedids, apks, repodir, archive): - """Generate the repo index files. - - :param apps: fully populated apps list - :param apks: full populated apks list - :param repodir: the repo directory - :param archive: True if this is the archive repo, False if it's the - main one. - :param categories: list of categories - """ - - def _resolve_description_link(appid): - if appid in apps: - return ("fdroid.app:" + appid, apps[appid].Name) - raise MetaDataException("Cannot resolve app id " + appid) - - nosigningkey = False - if not options.nosign: - if 'repo_keyalias' not in config: - nosigningkey = True - logging.critical("'repo_keyalias' not found in config.py!") - if 'keystore' not in config: - nosigningkey = True - logging.critical("'keystore' not found in config.py!") - if 'keystorepass' not in config and 'keystorepassfile' not in config: - nosigningkey = True - logging.critical("'keystorepass' not found in config.py!") - if 'keypass' not in config and 'keypassfile' not in config: - nosigningkey = True - logging.critical("'keypass' not found in config.py!") - if not os.path.exists(config['keystore']): - nosigningkey = True - logging.critical("'" + config['keystore'] + "' does not exist!") - if nosigningkey: - logging.warning("`fdroid update` requires a signing key, you can create one using:") - logging.warning("\tfdroid update --create-key") - sys.exit(1) - - repodict = collections.OrderedDict() - repodict['timestamp'] = datetime.utcnow() - repodict['version'] = METADATA_VERSION - - if config['repo_maxage'] != 0: - repodict['maxage'] = config['repo_maxage'] - - if archive: - repodict['name'] = config['archive_name'] - repodict['icon'] = os.path.basename(config['archive_icon']) - repodict['address'] = config['archive_url'] - repodict['description'] = config['archive_description'] - urlbasepath = os.path.basename(urllib.parse.urlparse(config['archive_url']).path) - else: - repodict['name'] = config['repo_name'] - repodict['icon'] = os.path.basename(config['repo_icon']) - repodict['address'] = config['repo_url'] - repodict['description'] = config['repo_description'] - urlbasepath = os.path.basename(urllib.parse.urlparse(config['repo_url']).path) - - mirrorcheckfailed = False - mirrors = [] - for mirror in sorted(config.get('mirrors', [])): - base = os.path.basename(urllib.parse.urlparse(mirror).path.rstrip('/')) - if config.get('nonstandardwebroot') is not True and base != 'fdroid': - logging.error("mirror '" + mirror + "' does not end with 'fdroid'!") - mirrorcheckfailed = True - # must end with / or urljoin strips a whole path segment - if mirror.endswith('/'): - mirrors.append(urllib.parse.urljoin(mirror, urlbasepath)) - else: - mirrors.append(urllib.parse.urljoin(mirror + '/', urlbasepath)) - for mirror in config.get('servergitmirrors', []): - mirror = get_raw_mirror(mirror) - if mirror is not None: - mirrors.append(mirror + '/') - if mirrorcheckfailed: - sys.exit(1) - if mirrors: - repodict['mirrors'] = mirrors - - appsWithPackages = collections.OrderedDict() - for packageName in sortedids: - app = apps[packageName] - if app['Disabled']: - continue - - # only include apps with packages - for apk in apks: - if apk['packageName'] == packageName: - newapp = copy.copy(app) # update wiki needs unmodified description - newapp['Description'] = metadata.description_html(app['Description'], - _resolve_description_link) - appsWithPackages[packageName] = newapp - break - - requestsdict = dict() - for command in ('install', 'uninstall'): - packageNames = [] - key = command + '_list' - if key in config: - if isinstance(config[key], str): - packageNames = [config[key]] - elif all(isinstance(item, str) for item in config[key]): - packageNames = config[key] - else: - raise TypeError('only accepts strings, lists, and tuples') - requestsdict[command] = packageNames - - make_index_v0(appsWithPackages, apks, repodir, repodict, requestsdict) - make_index_v1(appsWithPackages, apks, repodir, repodict, requestsdict) - - -def make_index_v1(apps, packages, repodir, repodict, requestsdict): - - def _index_encoder_default(obj): - if isinstance(obj, set): - return list(obj) - if isinstance(obj, datetime): - return int(obj.timestamp() * 1000) # Java expects milliseconds - raise TypeError(repr(obj) + " is not JSON serializable") - - output = collections.OrderedDict() - output['repo'] = repodict - output['requests'] = requestsdict - - appslist = [] - output['apps'] = appslist - for appid, appdict in apps.items(): - d = collections.OrderedDict() - appslist.append(d) - for k, v in sorted(appdict.items()): - if not v: - continue - if k in ('builds', 'comments', 'metadatapath', - 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes', - 'Provides', 'Repo', 'RepoType', 'RequiresRoot', - 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode', - 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'): - continue - - # name things after the App class fields in fdroidclient - if k == 'id': - k = 'packageName' - elif k == 'CurrentVersionCode': # TODO make SuggestedVersionCode the canonical name - k = 'suggestedVersionCode' - elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name - k = 'suggestedVersionName' - elif k == 'AutoName': - if 'Name' not in apps[appid]: - d['name'] = v - continue - else: - k = k[:1].lower() + k[1:] - d[k] = v - - output_packages = dict() - output['packages'] = output_packages - for package in packages: - packageName = package['packageName'] - if packageName in output_packages: - packagelist = output_packages[packageName] - else: - packagelist = [] - output_packages[packageName] = packagelist - d = collections.OrderedDict() - packagelist.append(d) - for k, v in sorted(package.items()): - if not v: - continue - if k in ('icon', 'icons', 'icons_src', 'name', ): - continue - d[k] = v - - json_name = 'index-v1.json' - index_file = os.path.join(repodir, json_name) - with open(index_file, 'w') as fp: - json.dump(output, fp, default=_index_encoder_default) - - if options.nosign: - logging.debug('index-v1 must have a signature, use `fdroid signindex` to create it!') - else: - common.sign_index_v1(repodir, json_name) - - -def make_index_v0(apps, apks, repodir, repodict, requestsdict): - '''aka index.jar aka index.xml''' - - doc = Document() - - def addElement(name, value, doc, parent): - el = doc.createElement(name) - el.appendChild(doc.createTextNode(value)) - parent.appendChild(el) - - def addElementNonEmpty(name, value, doc, parent): - if not value: - return - addElement(name, value, doc, parent) - - def addElementIfInApk(name, apk, key, doc, parent): - if key not in apk: - return - value = str(apk[key]) - addElement(name, value, doc, parent) - - def addElementCDATA(name, value, doc, parent): - el = doc.createElement(name) - el.appendChild(doc.createCDATASection(value)) - parent.appendChild(el) - - root = doc.createElement("fdroid") - doc.appendChild(root) - - repoel = doc.createElement("repo") - - repoel.setAttribute("name", repodict['name']) - if 'maxage' in repodict: - repoel.setAttribute("maxage", str(repodict['maxage'])) - repoel.setAttribute("icon", os.path.basename(repodict['icon'])) - repoel.setAttribute("url", repodict['address']) - addElement('description', repodict['description'], doc, repoel) - for mirror in repodict.get('mirrors', []): - addElement('mirror', mirror, doc, repoel) - - repoel.setAttribute("version", str(repodict['version'])) - repoel.setAttribute("timestamp", '%d' % repodict['timestamp'].timestamp()) - - repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8')) - root.appendChild(repoel) - - for command in ('install', 'uninstall'): - for packageName in requestsdict[command]: - element = doc.createElement(command) - root.appendChild(element) - element.setAttribute('packageName', packageName) - - for appid, appdict in apps.items(): - app = metadata.App(appdict) - - if app.Disabled is not None: - continue - - # Get a list of the apks for this app... - apklist = [] - for apk in apks: - if apk['packageName'] == appid: - apklist.append(apk) - - if len(apklist) == 0: - continue - - apel = doc.createElement("application") - apel.setAttribute("id", app.id) - root.appendChild(apel) - - addElement('id', app.id, doc, apel) - if app.added: - addElement('added', app.added.strftime('%Y-%m-%d'), doc, apel) - if app.lastUpdated: - addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel) - addElement('name', app.Name, doc, apel) - addElement('summary', app.Summary, doc, apel) - if app.icon: - addElement('icon', app.icon, doc, apel) - - if app.get('Description'): - description = app.Description - else: - description = '

No description available

' - addElement('desc', description, doc, apel) - addElement('license', app.License, doc, apel) - if app.Categories: - addElement('categories', ','.join(app.Categories), doc, apel) - # We put the first (primary) category in LAST, which will have - # the desired effect of making clients that only understand one - # category see that one. - addElement('category', app.Categories[0], doc, apel) - addElement('web', app.WebSite, doc, apel) - addElement('source', app.SourceCode, doc, apel) - addElement('tracker', app.IssueTracker, doc, apel) - addElementNonEmpty('changelog', app.Changelog, doc, apel) - addElementNonEmpty('author', app.AuthorName, doc, apel) - addElementNonEmpty('email', app.AuthorEmail, doc, apel) - addElementNonEmpty('donate', app.Donate, doc, apel) - addElementNonEmpty('bitcoin', app.Bitcoin, doc, apel) - addElementNonEmpty('litecoin', app.Litecoin, doc, apel) - addElementNonEmpty('flattr', app.FlattrID, doc, apel) - - # These elements actually refer to the current version (i.e. which - # one is recommended. They are historically mis-named, and need - # changing, but stay like this for now to support existing clients. - addElement('marketversion', app.CurrentVersion, doc, apel) - addElement('marketvercode', app.CurrentVersionCode, doc, apel) - - if app.Provides: - pv = app.Provides.split(',') - addElementNonEmpty('provides', ','.join(pv), doc, apel) - if app.RequiresRoot: - addElement('requirements', 'root', doc, apel) - - # Sort the apk list into version order, just so the web site - # doesn't have to do any work by default... - apklist = sorted(apklist, key=lambda apk: apk['versionCode'], reverse=True) - - if 'antiFeatures' in apklist[0]: - app.AntiFeatures.extend(apklist[0]['antiFeatures']) - if app.AntiFeatures: - addElementNonEmpty('antifeatures', ','.join(app.AntiFeatures), doc, apel) - - # Check for duplicates - they will make the client unhappy... - for i in range(len(apklist) - 1): - if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']: - logging.critical("duplicate versions: '%s' - '%s'" % ( - apklist[i]['apkName'], apklist[i + 1]['apkName'])) - sys.exit(1) - - current_version_code = 0 - current_version_file = None - for apk in apklist: - file_extension = common.get_file_extension(apk['apkName']) - # find the APK for the "Current Version" - if current_version_code < apk['versionCode']: - current_version_code = apk['versionCode'] - if current_version_code < int(app.CurrentVersionCode): - current_version_file = apk['apkName'] - - apkel = doc.createElement("package") - apel.appendChild(apkel) - addElement('version', apk['versionName'], doc, apkel) - addElement('versioncode', str(apk['versionCode']), doc, apkel) - addElement('apkname', apk['apkName'], doc, apkel) - addElementIfInApk('srcname', apk, 'srcname', doc, apkel) - - hashel = doc.createElement("hash") - hashel.setAttribute('type', 'sha256') - hashel.appendChild(doc.createTextNode(apk['hash'])) - apkel.appendChild(hashel) - - addElement('size', str(apk['size']), doc, apkel) - addElementIfInApk('sdkver', apk, - 'minSdkVersion', doc, apkel) - addElementIfInApk('targetSdkVersion', apk, - 'targetSdkVersion', doc, apkel) - addElementIfInApk('maxsdkver', apk, - 'maxSdkVersion', doc, apkel) - addElementIfInApk('obbMainFile', apk, - 'obbMainFile', doc, apkel) - addElementIfInApk('obbMainFileSha256', apk, - 'obbMainFileSha256', doc, apkel) - addElementIfInApk('obbPatchFile', apk, - 'obbPatchFile', doc, apkel) - addElementIfInApk('obbPatchFileSha256', apk, - 'obbPatchFileSha256', doc, apkel) - if 'added' in apk: - addElement('added', apk['added'].strftime('%Y-%m-%d'), doc, apkel) - - if file_extension == 'apk': # sig is required for APKs, but only APKs - addElement('sig', apk['sig'], doc, apkel) - - old_permissions = set() - sorted_permissions = sorted(apk['uses-permission']) - for perm in sorted_permissions: - perm_name = perm.name - if perm_name.startswith("android.permission."): - perm_name = perm_name[19:] - old_permissions.add(perm_name) - addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel) - - for permission in sorted_permissions: - permel = doc.createElement('uses-permission') - permel.setAttribute('name', permission.name) - if permission.maxSdkVersion is not None: - permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion) - apkel.appendChild(permel) - for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']): - permel = doc.createElement('uses-permission-sdk-23') - permel.setAttribute('name', permission_sdk_23.name) - if permission_sdk_23.maxSdkVersion is not None: - permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion) - apkel.appendChild(permel) - if 'nativecode' in apk: - addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel) - addElementNonEmpty('features', ','.join(sorted(apk['features'])), doc, apkel) - - if current_version_file is not None \ - and config['make_current_version_link'] \ - and repodir == 'repo': # only create these - namefield = config['current_version_name_source'] - sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield)) - apklinkname = sanitized_name + '.apk' - current_version_path = os.path.join(repodir, current_version_file) - if os.path.islink(apklinkname): - os.remove(apklinkname) - os.symlink(current_version_path, apklinkname) - # also symlink gpg signature, if it exists - for extension in ('.asc', '.sig'): - sigfile_path = current_version_path + extension - if os.path.exists(sigfile_path): - siglinkname = apklinkname + extension - if os.path.islink(siglinkname): - os.remove(siglinkname) - os.symlink(sigfile_path, siglinkname) - - if options.pretty: - output = doc.toprettyxml(encoding='utf-8') - else: - output = doc.toxml(encoding='utf-8') - - with open(os.path.join(repodir, 'index.xml'), 'wb') as f: - f.write(output) - - if 'repo_keyalias' in config: - - if options.nosign: - logging.info("Creating unsigned index in preparation for signing") - else: - logging.info("Creating signed index with this key (SHA256):") - logging.info("%s" % repo_pubkey_fingerprint) - - # Create a jar of the index... - jar_output = 'index_unsigned.jar' if options.nosign else 'index.jar' - p = FDroidPopen(['jar', 'cf', jar_output, 'index.xml'], cwd=repodir) - if p.returncode != 0: - logging.critical("Failed to create {0}".format(jar_output)) - sys.exit(1) - - # Sign the index... - signed = os.path.join(repodir, 'index.jar') - if options.nosign: - # Remove old signed index if not signing - if os.path.exists(signed): - os.remove(signed) - else: - common.signjar(signed) - - # Copy the repo icon into the repo directory... - icon_dir = os.path.join(repodir, 'icons') - iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon'])) - shutil.copyfile(config['repo_icon'], iconfilename) - - def make_categories_txt(repodir, categories): '''Write a category list in the repo to allow quick access''' catdata = '' @@ -1984,7 +1490,7 @@ def main(): appdict = dict() appdict[appid] = app if os.path.isdir(repodir): - make_index(appdict, [appid], apks, repodir, False) + index.make(appdict, [appid], apks, repodir, False) else: logging.info('Skipping index generation for ' + appid) return @@ -1993,13 +1499,13 @@ def main(): archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) # Make the index for the main repo... - make_index(apps, sortedids, apks, repodirs[0], False) + index.make(apps, sortedids, apks, repodirs[0], False) make_categories_txt(repodirs[0], categories) # If there's an archive repo, make the index for it. We already scanned it # earlier on. if len(repodirs) > 1: - make_index(apps, sortedids, archapks, repodirs[1], True) + index.make(apps, sortedids, archapks, repodirs[1], True) if config.get('binary_transparency_remote'): make_binary_transparency_log(repodirs) diff --git a/tests/common.TestCase b/tests/common.TestCase index db0be6d0..50cbd320 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -12,6 +12,8 @@ import tempfile import unittest from zipfile import ZipFile +import fdroidserver.signindex + localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) print('localmodule: ' + localmodule) @@ -163,6 +165,7 @@ class CommonTest(unittest.TestCase): config = fdroidserver.common.read_config(fdroidserver.common.options) config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config + fdroidserver.signindex.config = config basedir = os.path.dirname(__file__) tmpdir = os.path.join(basedir, '..', '.testfiles') @@ -174,7 +177,7 @@ class CommonTest(unittest.TestCase): sourcefile = os.path.join(sourcedir, f) testfile = os.path.join(testsdir, f) shutil.copy(sourcefile, testsdir) - fdroidserver.common.signjar(testfile) + fdroidserver.signindex.sign_jar(testfile) # these should be resigned, and therefore different self.assertNotEqual(open(sourcefile, 'rb').read(), open(testfile, 'rb').read())