fdroidserver/fdroidserver/server.py

#!/usr/bin/env python3
#
# server.py - part of the FDroid server tools
# Copyright (C) 2010-15, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import glob
import hashlib
import os
import paramiko
import pwd
import subprocess
from argparse import ArgumentParser
import logging
import shutil

from . import common

config = None
options = None


def update_awsbucket(repo_section):
    '''
    Upload the contents of the directory `repo_section` (including
    subdirectories) to the AWS S3 "bucket". The contents of that subdir of the
    bucket will first be deleted.

    Requires AWS credentials set in config.py: awsaccesskeyid, awssecretkey
    '''

    logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
                  + config['awsbucket'] + '"')

    import libcloud.security
    libcloud.security.VERIFY_SSL_CERT = True
    from libcloud.storage.types import Provider, ContainerDoesNotExistError
    from libcloud.storage.providers import get_driver

    if not config.get('awsaccesskeyid') or not config.get('awssecretkey'):
        logging.error('To use awsbucket, you must set awssecretkey and awsaccesskeyid in config.py!')
        sys.exit(1)
    awsbucket = config['awsbucket']

    cls = get_driver(Provider.S3)
    driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
    try:
        container = driver.get_container(container_name=awsbucket)
    except ContainerDoesNotExistError:
        container = driver.create_container(container_name=awsbucket)
        logging.info('Created new container "' + container.name + '"')

    upload_dir = 'fdroid/' + repo_section
    objs = dict()
    for obj in container.list_objects():
        if obj.name.startswith(upload_dir + '/'):
            objs[obj.name] = obj

    for root, _, files in os.walk(os.path.join(os.getcwd(), repo_section)):
        for name in files:
            upload = False
            file_to_upload = os.path.join(root, name)
            object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
            if object_name not in objs:
                upload = True
            else:
                obj = objs.pop(object_name)
                if obj.size != os.path.getsize(file_to_upload):
                    upload = True
                else:
                    # if the sizes match, then compare by MD5
                    md5 = hashlib.md5()
                    with open(file_to_upload, 'rb') as f:
                        while True:
                            data = f.read(8192)
                            if not data:
                                break
                            md5.update(data)
                    if obj.hash != md5.hexdigest():
                        s3url = 's3://' + awsbucket + '/' + obj.name
                        logging.info(' deleting ' + s3url)
                        if not driver.delete_object(obj):
                            logging.warn('Could not delete ' + s3url)
                        upload = True

            if upload:
                logging.debug(' uploading "' + file_to_upload + '"...')
                extra = {'acl': 'public-read'}
                if file_to_upload.endswith('.sig'):
                    extra['content_type'] = 'application/pgp-signature'
                elif file_to_upload.endswith('.asc'):
                    extra['content_type'] = 'application/pgp-signature'
                logging.info(' uploading ' + os.path.relpath(file_to_upload)
                             + ' to s3://' + awsbucket + '/' + object_name)
                with open(file_to_upload, 'rb') as iterator:
                    obj = driver.upload_object_via_stream(iterator=iterator,
                                                          container=container,
                                                          object_name=object_name,
                                                          extra=extra)
    # delete the remnants in the bucket, they do not exist locally
    while objs:
        object_name, obj = objs.popitem()
        s3url = 's3://' + awsbucket + '/' + object_name
        if object_name.startswith(upload_dir):
            logging.warn(' deleting ' + s3url)
            driver.delete_object(obj)
        else:
            logging.info(' skipping ' + s3url)


def update_serverwebroot(serverwebroot, repo_section):
    # use a checksum comparison for accurate comparisons on different
    # filesystems, for example, FAT has a low resolution timestamp
    rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
    if not options.no_checksum:
        rsyncargs.append('--checksum')
    if options.verbose:
        rsyncargs += ['--verbose']
    if options.quiet:
        rsyncargs += ['--quiet']
    if options.identity_file is not None:
        rsyncargs += ['-e', 'ssh -i ' + options.identity_file]
    if 'identity_file' in config:
        rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
    indexxml = os.path.join(repo_section, 'index.xml')
    indexjar = os.path.join(repo_section, 'index.jar')
    # Upload the first time without the index files and delay the deletion as
    # much as possible, that keeps the repo functional while this update is
    # running.  Then once it is complete, rerun the command again to upload
    # the index files.  Always using the same target with rsync allows for
    # very strict settings on the receiving server, you can literally specify
    # the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
    # (serverwebroot is guaranteed to have a trailing slash in common.py)
    logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
    if subprocess.call(rsyncargs +
                       ['--exclude', indexxml, '--exclude', indexjar,
                        repo_section, serverwebroot]) != 0:
        sys.exit(1)
    if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
        sys.exit(1)
    # upload "current version" symlinks if requested
    if config['make_current_version_link'] and repo_section == 'repo':
        links_to_upload = []
        for f in glob.glob('*.apk') \
                + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
            if os.path.islink(f):
                links_to_upload.append(f)
        if len(links_to_upload) > 0:
            if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
                sys.exit(1)


def _local_sync(fromdir, todir):
    rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
                 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
    # use stricter rsync checking on all files since people using offline mode
    # are already prioritizing security above ease and speed
    if not options.no_checksum:
        rsyncargs.append('--checksum')
    if options.verbose:
        rsyncargs += ['--verbose']
    if options.quiet:
        rsyncargs += ['--quiet']
    logging.debug(' '.join(rsyncargs + [fromdir, todir]))
    if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
        sys.exit(1)


def sync_from_localcopy(repo_section, local_copy_dir):
    logging.info('Syncing from local_copy_dir to this repo.')
    # trailing slashes have a meaning in rsync which is not needed here, so
    # make sure both paths have exactly one trailing slash
    _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
                repo_section.rstrip('/') + '/')


def update_localcopy(repo_section, local_copy_dir):
    # local_copy_dir is guaranteed to have a trailing slash in main() below
    _local_sync(repo_section, local_copy_dir)


def update_servergitmirrors(servergitmirrors, repo_section):
    # depend on GitPython only if users set a git mirror
    import git
    # right now we support only 'repo' git-mirroring
    if repo_section == 'repo':
        # create a new git-mirror folder
        repo_dir = os.path.join('.', 'git-mirror/')

        # remove if already present
        if os.path.isdir(repo_dir):
            shutil.rmtree(repo_dir)

        repo = git.Repo.init(repo_dir)

        # take care of each mirror
        for mirror in servergitmirrors:
            hostname = mirror.split("/")[2]
            repo.create_remote(hostname, mirror)
            logging.info('Mirroring to: ' + mirror)

        # copy local 'repo' to 'git-mirror/fdroid/repo directory' with _local_sync
        fdroid_repo_path = os.path.join(repo_dir, "fdroid")
        _local_sync(repo_section, fdroid_repo_path)

        # sadly index.add don't allow the --all parameter
        repo.git.add(all=True)
        repo.index.commit("fdroidserver git-mirror")

        # push for every remote. This will overwrite the git history
        for remote in repo.remotes:
            remote.push('master', force=True, set_upstream=True)


def upload_to_android_observatory(repo_section):
    # depend on requests and lxml only if users enable AO
    import requests
    from lxml.html import fromstring

    if repo_section == 'repo':
        for f in glob.glob(os.path.join(repo_section, '*.apk')):
            fpath = f
            fname = os.path.basename(f)
            logging.info('Uploading ' + fname + ' to androidobservatory.org')

            # upload the file with a post request
            r = requests.post('https://androidobservatory.org/upload', files={'apk': (fname, open(fpath, 'rb'))})
            response = r.text
            page = r.url

            # from now on XPath will be used to retrieve the message in the HTML
            # androidobservatory doesn't have a nice API to talk with
            # so we must scrape the page content
            tree = fromstring(response)
            alert = tree.xpath("//html/body/div[@class='container content-container']/div[@class='alert alert-info']")[0]

            message = ""
            appurl = page
            for el in alert:
                # if the application was added successfully we retrive the url
                # if the application was already uploaded we use the redirect page url
                if el.attrib.get("href") is not None:
                    appurl = page + el.attrib["href"][1:]
                    message += el.text.replace(" here", "") + el.tail
                else:
                    message += el.tail
            message = message.strip() + " " + appurl
            logging.info(message)


def upload_to_virustotal(repo_section, vt_apikey):
    import requests

    if repo_section == 'repo':
        for f in glob.glob(os.path.join(repo_section, '*.apk')):
            fpath = f
            fname = os.path.basename(f)
            logging.info('Uploading ' + fname + ' to virustotal.com')

            # upload the file with a post request
            params = {'apikey': vt_apikey}
            files = {'file': (fname, open(fpath, 'rb'))}
            r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=params)
            response = r.json()

            logging.info(response['verbose_msg'] + " " + response['permalink'])


def main():
    global config, options

    # Parse command line...
    parser = ArgumentParser()
    common.setup_global_opts(parser)
    parser.add_argument("command", help="command to execute, either 'init' or 'update'")
    parser.add_argument("-i", "--identity-file", default=None,
                        help="Specify an identity file to provide to SSH for rsyncing")
    parser.add_argument("--local-copy-dir", default=None,
                        help="Specify a local folder to sync the repo to")
    parser.add_argument("--sync-from-local-copy-dir", action="store_true", default=False,
                        help="Before uploading to servers, sync from local copy dir")
    parser.add_argument("--no-checksum", action="store_true", default=False,
                        help="Don't use rsync checksums")
    options = parser.parse_args()

    config = common.read_config(options)

    if options.command != 'init' and options.command != 'update':
        logging.critical("The only commands currently supported are 'init' and 'update'")
        sys.exit(1)

    if config.get('nonstandardwebroot') is True:
        standardwebroot = False
    else:
        standardwebroot = True

    for serverwebroot in config.get('serverwebroot', []):
        # this supports both an ssh host:path and just a path
        s = serverwebroot.rstrip('/').split(':')
        if len(s) == 1:
            fdroiddir = s[0]
        elif len(s) == 2:
            host, fdroiddir = s
        else:
            logging.error('Malformed serverwebroot line: ' + serverwebroot)
            sys.exit(1)
        repobase = os.path.basename(fdroiddir)
        if standardwebroot and repobase != 'fdroid':
            logging.error('serverwebroot path does not end with "fdroid", '
                          + 'perhaps you meant one of these:\n\t'
                          + serverwebroot.rstrip('/') + '/fdroid\n\t'
                          + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
            sys.exit(1)

    if options.local_copy_dir is not None:
        local_copy_dir = options.local_copy_dir
    elif config.get('local_copy_dir'):
        local_copy_dir = config['local_copy_dir']
    else:
        local_copy_dir = None
    if local_copy_dir is not None:
        fdroiddir = local_copy_dir.rstrip('/')
        if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
            logging.error('local_copy_dir must be directory, not a file!')
            sys.exit(1)
        if not os.path.exists(os.path.dirname(fdroiddir)):
            logging.error('The root dir for local_copy_dir "'
                          + os.path.dirname(fdroiddir)
                          + '" does not exist!')
            sys.exit(1)
        if not os.path.isabs(fdroiddir):
            logging.error('local_copy_dir must be an absolute path!')
            sys.exit(1)
        repobase = os.path.basename(fdroiddir)
        if standardwebroot and repobase != 'fdroid':
            logging.error('local_copy_dir does not end with "fdroid", '
                          + 'perhaps you meant: ' + fdroiddir + '/fdroid')
            sys.exit(1)
        if local_copy_dir[-1] != '/':
            local_copy_dir += '/'
        local_copy_dir = local_copy_dir.replace('//', '/')
        if not os.path.exists(fdroiddir):
            os.mkdir(fdroiddir)

    if not config.get('awsbucket') \
            and not config.get('serverwebroot') \
            and not config.get('servergitmirrors') \
            and not config.get('uploadto_androidobservatory') \
            and not config.get('virustotal_apikey') \
            and local_copy_dir is None:
        logging.warn('No option set! Edit your config.py to set at least one among:\n'
                     + 'serverwebroot, servergitmirrors, local_copy_dir, awsbucket, virustotal_apikey or uploadto_androidobservatory')
        sys.exit(1)

    repo_sections = ['repo']
    if config['archive_older'] != 0:
        repo_sections.append('archive')
        if not os.path.exists('archive'):
            os.mkdir('archive')
    if config['per_app_repos']:
        repo_sections += common.get_per_app_repos()

    if options.command == 'init':
        ssh = paramiko.SSHClient()
        ssh.load_system_host_keys()
        for serverwebroot in config.get('serverwebroot', []):
            sshstr, remotepath = serverwebroot.rstrip('/').split(':')
            if sshstr.find('@') >= 0:
                username, hostname = sshstr.split('@')
            else:
                username = pwd.getpwuid(os.getuid())[0]  # get effective uid
                hostname = sshstr
            ssh.connect(hostname, username=username)
            sftp = ssh.open_sftp()
            if os.path.basename(remotepath) \
                    not in sftp.listdir(os.path.dirname(remotepath)):
                sftp.mkdir(remotepath, mode=0o755)
            for repo_section in repo_sections:
                repo_path = os.path.join(remotepath, repo_section)
                if os.path.basename(repo_path) \
                        not in sftp.listdir(remotepath):
                    sftp.mkdir(repo_path, mode=0o755)
            sftp.close()
            ssh.close()
    elif options.command == 'update':
        for repo_section in repo_sections:
            if local_copy_dir is not None:
                if config['sync_from_local_copy_dir'] and os.path.exists(repo_section):
                    sync_from_localcopy(repo_section, local_copy_dir)
                else:
                    update_localcopy(repo_section, local_copy_dir)
            for serverwebroot in config.get('serverwebroot', []):
                update_serverwebroot(serverwebroot, repo_section)
            if config.get('servergitmirrors', []):
                # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
                servergitmirrors = config.get('servergitmirrors', [])
                update_servergitmirrors(servergitmirrors, repo_section)
            if config.get('awsbucket'):
                update_awsbucket(repo_section)
            if config.get('uploadto_androidobservatory'):
                upload_to_android_observatory(repo_section)
            if config.get('virustotal_apikey'):
                upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
    sys.exit(0)


if __name__ == "__main__":
    main()