fdroidserver/fdroidserver/common.py

# -*- coding: utf-8 -*-
#
# common.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import glob, os, sys, re
import shutil
import stat
import subprocess
import time
import operator
import cgi
import Queue
import threading
import magic

config = None
options = None

# These can only contain 'yes' or 'no'
bool_keys = (
        'submodules', 'oldsdkloc',
        'forceversion', 'forcevercode',
        'fixtrans', 'fixapos', 'novcheck')

def read_config(opts, config_file='config.py'):
    """Read the repository config

    The config is read from config_file, which is in the current directory when
    any of the repo management commands are used.
    """
    global config, options

    if config is not None:
        return config
    if not os.path.isfile(config_file):
        print "Missing config file - is this a repo directory?"
        sys.exit(2)
    st = os.stat(config_file)
    if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
        print("WARNING: unsafe permissions on config.py (should be 0600)!")

    options = opts
    if not hasattr(options, 'verbose'):
        options.verbose = False

    config = {
        'build_server_always': False,
        'mvn3': "mvn3",
        'archive_older': 0,
        'gradle': 'gradle',
        'update_stats': False,
        'archive_older': 0,
        'max_icon_size': 72,
        'stats_to_carbon': False
    }
    if options.verbose:
        print "Reading %s..." % config_file
    execfile(config_file, config)
    return config


def getvcs(vcstype, remote, local):
    if vcstype == 'git':
        return vcs_git(remote, local)
    if vcstype == 'svn':
        return vcs_svn(remote, local)
    if vcstype == 'git-svn':
        return vcs_gitsvn(remote, local)
    if vcstype == 'hg':
        return vcs_hg(remote, local)
    if vcstype == 'bzr':
        return vcs_bzr(remote, local)
    if vcstype == 'srclib':
        if local != 'build/srclib/' + remote:
            raise VCSException("Error: srclib paths are hard-coded!")
        return getsrclib(remote, 'build/srclib', raw=True)
    raise VCSException("Invalid vcs type " + vcstype)

def getsrclibvcs(name):
    srclib_path = os.path.join('srclibs', name + ".txt")
    if not os.path.exists(srclib_path):
        raise VCSException("Missing srclib " + name)
    return parse_srclib(srclib_path)['Repo Type']

class vcs:
    def __init__(self, remote, local):

        # svn, git-svn and bzr may require auth
        self.username = None
        if self.repotype() in ('svn', 'git-svn', 'bzr'):
            if '@' in remote:
                self.username, remote = remote.split('@')
                if ':' not in self.username:
                    raise VCSException("Password required with username")
                self.username, self.password = self.username.split(':')

        self.remote = remote
        self.local = local
        self.refreshed = False
        self.srclib = None

    # Take the local repository to a clean version of the given revision, which
    # is specificed in the VCS's native format. Beforehand, the repository can
    # be dirty, or even non-existent. If the repository does already exist
    # locally, it will be updated from the origin, but only once in the
    # lifetime of the vcs object.
    # None is acceptable for 'rev' if you know you are cloning a clean copy of
    # the repo - otherwise it must specify a valid revision.
    def gotorevision(self, rev):

        # The .fdroidvcs-id file for a repo tells us what VCS type
        # and remote that directory was created from, allowing us to drop it
        # automatically if either of those things changes.
        fdpath = os.path.join(self.local, '..',
                '.fdroidvcs-' + os.path.basename(self.local))
        cdata = self.repotype() + ' ' + self.remote
        writeback = True
        deleterepo = False
        if os.path.exists(self.local):
            if os.path.exists(fdpath):
                with open(fdpath, 'r') as f:
                    fsdata = f.read()
                if fsdata == cdata:
                    writeback = False
                else:
                    deleterepo = True
                    print "*** Repository details changed - deleting ***"
            else:
                deleterepo = True
                print "*** Repository details missing - deleting ***"
        if deleterepo:
            shutil.rmtree(self.local)

        self.gotorevisionx(rev)

        # If necessary, write the .fdroidvcs file.
        if writeback:
            with open(fdpath, 'w') as f:
                f.write(cdata)

    # Derived classes need to implement this. It's called once basic checking
    # has been performend.
    def gotorevisionx(self, rev):
        raise VCSException("This VCS type doesn't define gotorevisionx")

    # Initialise and update submodules
    def initsubmodules(self):
        raise VCSException('Submodules not supported for this vcs type')

    # Get a list of all known tags
    def gettags(self):
        raise VCSException('gettags not supported for this vcs type')

    # Get current commit reference (hash, revision, etc)
    def getref(self):
        raise VCSException('getref not supported for this vcs type')

    # Returns the srclib (name, path) used in setting up the current
    # revision, or None.
    def getsrclib(self):
        return self.srclib

class vcs_git(vcs):

    def repotype(self):
        return 'git'

    # If the local directory exists, but is somehow not a git repository, git
    # will traverse up the directory tree until it finds one that is (i.e.
    # fdroidserver) and then we'll proceed to destroy it! This is called as
    # a safety check.
    def checkrepo(self):
        p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
                stdout=subprocess.PIPE, cwd=self.local)
        result = p.communicate()[0].rstrip()
        if not result.endswith(self.local):
            raise VCSException('Repository mismatch')

    def gotorevisionx(self, rev):
        if not os.path.exists(self.local):
            # Brand new checkout...
            if subprocess.call(['git', 'clone', self.remote, self.local]) != 0:
                raise VCSException("Git clone failed")
            self.checkrepo()
        else:
            self.checkrepo()
            # Discard any working tree changes...
            if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
                raise VCSException("Git reset failed")
            # Remove untracked files now, in case they're tracked in the target
            # revision (it happens!)...
            if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
                raise VCSException("Git clean failed")
            if not self.refreshed:
                # Get latest commits and tags from remote...
                if subprocess.call(['git', 'fetch', 'origin'],
                        cwd=self.local) != 0:
                    raise VCSException("Git fetch failed")
                if subprocess.call(['git', 'fetch', '--tags', 'origin'],
                        cwd=self.local) != 0:
                    raise VCSException("Git fetch failed")
                self.refreshed = True
        # Check out the appropriate revision...
        rev = str(rev if rev else 'origin/master')
        if subprocess.call(['git', 'checkout', '-f', rev], cwd=self.local) != 0:
            raise VCSException("Git checkout failed")
        # Get rid of any uncontrolled files left behind...
        if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
            raise VCSException("Git clean failed")

    def initsubmodules(self):
        self.checkrepo()
        if subprocess.call(['git', 'submodule', 'init'],
                cwd=self.local) != 0:
            raise VCSException("Git submodule init failed")
        if subprocess.call(['git', 'submodule', 'update'],
                cwd=self.local) != 0:
            raise VCSException("Git submodule update failed")
        if subprocess.call(['git', 'submodule', 'foreach',
            'git', 'reset', '--hard'],
                cwd=self.local) != 0:
            raise VCSException("Git submodule reset failed")
        if subprocess.call(['git', 'submodule', 'foreach',
            'git', 'clean', '-dffx'],
                cwd=self.local) != 0:
            raise VCSException("Git submodule clean failed")

    def gettags(self):
        self.checkrepo()
        p = subprocess.Popen(['git', 'tag'],
                stdout=subprocess.PIPE, cwd=self.local)
        return p.communicate()[0].splitlines()


class vcs_gitsvn(vcs):

    def repotype(self):
        return 'git-svn'

    # Damn git-svn tries to use a graphical password prompt, so we have to
    # trick it into taking the password from stdin
    def userargs(self):
        if self.username is None:
            return ('', '')
        return ('echo "%s" | DISPLAY="" ' % self.password, '--username "%s"' % self.username)

    # If the local directory exists, but is somehow not a git repository, git
    # will traverse up the directory tree until it finds one that is (i.e.
    # fdroidserver) and then we'll proceed to destory it! This is called as
    # a safety check.
    def checkrepo(self):
        p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
                stdout=subprocess.PIPE, cwd=self.local)
        result = p.communicate()[0].rstrip()
        if not result.endswith(self.local):
            raise VCSException('Repository mismatch')

    def gotorevisionx(self, rev):
        if not os.path.exists(self.local):
            # Brand new checkout...
            gitsvn_cmd = '%sgit svn clone %s' % self.userargs()
            if ';' in self.remote:
                remote_split = self.remote.split(';')
                for i in remote_split[1:]:
                    if i.startswith('trunk='):
                        gitsvn_cmd += ' -T %s' % i[6:]
                    elif i.startswith('tags='):
                        gitsvn_cmd += ' -t %s' % i[5:]
                    elif i.startswith('branches='):
                        gitsvn_cmd += ' -b %s' % i[9:]
                if subprocess.call([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)],
                        shell=True) != 0:
                    raise VCSException("Git clone failed")
            else:
                if subprocess.call([gitsvn_cmd + " %s %s" % (self.remote, self.local)],
                        shell=True) != 0:
                    raise VCSException("Git clone failed")
            self.checkrepo()
        else:
            self.checkrepo()
            # Discard any working tree changes...
            if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
                raise VCSException("Git reset failed")
            # Remove untracked files now, in case they're tracked in the target
            # revision (it happens!)...
            if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
                raise VCSException("Git clean failed")
            if not self.refreshed:
                # Get new commits and tags from repo...
                if subprocess.call(['%sgit svn rebase %s' % self.userargs()],
                        cwd=self.local, shell=True) != 0:
                    raise VCSException("Git svn rebase failed")
                self.refreshed = True

        rev = str(rev if rev else 'master')
        if rev:
            nospaces_rev = rev.replace(' ', '%20')
            # Try finding a svn tag
            p = subprocess.Popen(['git', 'checkout', 'tags/' + nospaces_rev],
                    cwd=self.local, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out, err = p.communicate()
            if p.returncode == 0:
                print out
            else:
                # No tag found, normal svn rev translation
                # Translate svn rev into git format
                p = subprocess.Popen(['git', 'svn', 'find-rev', 'r' + rev],
                    cwd=self.local, stdout=subprocess.PIPE)
                git_rev = p.communicate()[0].rstrip()
                if p.returncode != 0 or not git_rev:
                    # Try a plain git checkout as a last resort
                    p = subprocess.Popen(['git', 'checkout', rev], cwd=self.local,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    out, err = p.communicate()
                    if p.returncode == 0:
                        print out
                    else:
                        raise VCSException("No git treeish found and direct git checkout failed")
                else:
                    # Check out the git rev equivalent to the svn rev
                    p = subprocess.Popen(['git', 'checkout', git_rev], cwd=self.local,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    out, err = p.communicate()
                    if p.returncode == 0:
                        print out
                    else:
                        raise VCSException("Git svn checkout failed")
        # Get rid of any uncontrolled files left behind...
        if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
            raise VCSException("Git clean failed")

    def gettags(self):
        self.checkrepo()
        return os.listdir(os.path.join(self.local, '.git/svn/refs/remotes/tags'))

    def getref(self):
        self.checkrepo()
        p = subprocess.Popen(['git', 'svn', 'find-rev', 'HEAD'],
                stdout=subprocess.PIPE, cwd=self.local)
        return p.communicate()[0].strip()

class vcs_svn(vcs):

    def repotype(self):
        return 'svn'

    def userargs(self):
        if self.username is None:
            return ['--non-interactive']
        return ['--username', self.username, 
                '--password', self.password,
                '--non-interactive']

    def gotorevisionx(self, rev):
        if not os.path.exists(self.local):
            if subprocess.call(['svn', 'checkout', self.remote, self.local] +
                    self.userargs()) != 0:
                raise VCSException("Svn checkout failed")
        else:
            for svncommand in (
                    'svn revert -R .',
                    r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
                if subprocess.call(svncommand, cwd=self.local, shell=True) != 0:
                    raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local))
            if not self.refreshed:
                if subprocess.call(['svn', 'update'] +
                        self.userargs(), cwd=self.local) != 0:
                    raise VCSException("Svn update failed")
                self.refreshed = True

        revargs = list(['-r', rev] if rev else [])
        if subprocess.call(['svn', 'update', '--force'] + revargs +
                self.userargs(), cwd=self.local) != 0:
            raise VCSException("Svn update failed")

    def getref(self):
        p = subprocess.Popen(['svn', 'info'],
                stdout=subprocess.PIPE, cwd=self.local)
        for line in p.communicate()[0].splitlines():
            if line is not None and line.startswith('Last Changed Rev: '):
                return line[18:]

class vcs_hg(vcs):

    def repotype(self):
        return 'hg'

    def gotorevisionx(self, rev):
        if not os.path.exists(self.local):
            if subprocess.call(['hg', 'clone', self.remote, self.local]) !=0:
                raise VCSException("Hg clone failed")
        else:
            if subprocess.call('hg status -uS | xargs rm -rf',
                    cwd=self.local, shell=True) != 0:
                raise VCSException("Hg clean failed")
            if not self.refreshed:
                if subprocess.call(['hg', 'pull'],
                        cwd=self.local) != 0:
                    raise VCSException("Hg pull failed")
                self.refreshed = True

        rev = str(rev if rev else 'default')
        if not rev:
            return
        if subprocess.call(['hg', 'update', '-C', rev],
                cwd=self.local) != 0:
            raise VCSException("Hg checkout failed")

    def gettags(self):
        p = subprocess.Popen(['hg', 'tags', '-q'],
                stdout=subprocess.PIPE, cwd=self.local)
        return p.communicate()[0].splitlines()[1:]


class vcs_bzr(vcs):

    def repotype(self):
        return 'bzr'

    def gotorevisionx(self, rev):
        if not os.path.exists(self.local):
            if subprocess.call(['bzr', 'branch', self.remote, self.local]) != 0:
                raise VCSException("Bzr branch failed")
        else:
            if subprocess.call(['bzr', 'clean-tree', '--force',
                    '--unknown', '--ignored'], cwd=self.local) != 0:
                raise VCSException("Bzr revert failed")
            if not self.refreshed:
                if subprocess.call(['bzr', 'pull'],
                        cwd=self.local) != 0:
                    raise VCSException("Bzr update failed")
                self.refreshed = True

        revargs = list(['-r', rev] if rev else [])
        if subprocess.call(['bzr', 'revert'] + revargs,
                cwd=self.local) != 0:
            raise VCSException("Bzr revert failed")

    def gettags(self):
        p = subprocess.Popen(['bzr', 'tags'],
                stdout=subprocess.PIPE, cwd=self.local)
        return [tag.split('   ')[0].strip() for tag in
                p.communicate()[0].splitlines()]


# Get the type expected for a given metadata field.
def metafieldtype(name):
    if name in ['Description', 'Maintainer Notes']:
        return 'multiline'
    if name == 'Requires Root':
        return 'flag'
    if name == 'Build Version':
        return 'build'
    if name == 'Build':
        return 'buildv2'
    if name == 'Use Built':
        return 'obsolete'
    return 'string'


# Parse metadata for a single application.
#
#  'metafile' - the filename to read. The package id for the application comes
#               from this filename. Pass None to get a blank entry.
#
# Returns a dictionary containing all the details of the application. There are
# two major kinds of information in the dictionary. Keys beginning with capital
# letters correspond directory to identically named keys in the metadata file.
# Keys beginning with lower case letters are generated in one way or another,
# and are not found verbatim in the metadata.
#
# Known keys not originating from the metadata are:
#
#  'id'               - the application's package ID
#  'builds'           - a list of dictionaries containing build information
#                       for each defined build
#  'comments'         - a list of comments from the metadata file. Each is
#                       a tuple of the form (field, comment) where field is
#                       the name of the field it preceded in the metadata
#                       file. Where field is None, the comment goes at the
#                       end of the file. Alternatively, 'build:version' is
#                       for a comment before a particular build version.
#  'descriptionlines' - original lines of description as formatted in the
#                       metadata file.
#
def parse_metadata(metafile):

    def parse_buildline(lines):
        value = "".join(lines)
        parts = [p.replace("\\,", ",")
                 for p in re.split(r"(?<!\\),", value)]
        if len(parts) < 3:
            raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
        thisbuild = {}
        thisbuild['origlines'] = lines
        thisbuild['version'] = parts[0]
        thisbuild['vercode'] = parts[1]
        try:
            int(thisbuild['vercode'])
        except:
            raise MetaDataException("Invalid version code for build in " + metafile.name)
        if parts[2].startswith('!'):
            # For backwards compatibility, handle old-style disabling,
            # including attempting to extract the commit from the message
            thisbuild['disable'] = parts[2][1:]
            commit = 'unknown - see disabled'
            index = parts[2].rfind('at ')
            if index != -1:
                commit = parts[2][index+3:]
                if commit.endswith(')'):
                    commit = commit[:-1]
            thisbuild['commit'] = commit
        else:
            thisbuild['commit'] = parts[2]
        for p in parts[3:]:
            pk, pv = p.split('=', 1)
            thisbuild[pk.strip()] = pv

        return thisbuild

    def add_comments(key):
        if not curcomments:
            return
        for comment in curcomments:
            thisinfo['comments'].append((key, comment))
        del curcomments[:]


    thisinfo = {}
    if metafile:
        if not isinstance(metafile, file):
            metafile = open(metafile, "r")
        thisinfo['id'] = metafile.name[9:-4]
    else:
        thisinfo['id'] = None

    # Defaults for fields that come from metadata...
    thisinfo['Name'] = None
    thisinfo['Auto Name'] = ''
    thisinfo['Categories'] = 'None'
    thisinfo['Description'] = []
    thisinfo['Summary'] = ''
    thisinfo['License'] = 'Unknown'
    thisinfo['Web Site'] = ''
    thisinfo['Source Code'] = ''
    thisinfo['Issue Tracker'] = ''
    thisinfo['Donate'] = None
    thisinfo['FlattrID'] = None
    thisinfo['Bitcoin'] = None
    thisinfo['Litecoin'] = None
    thisinfo['Disabled'] = None
    thisinfo['AntiFeatures'] = None
    thisinfo['Archive Policy'] = None
    thisinfo['Update Check Mode'] = 'None'
    thisinfo['Vercode Operation'] = None
    thisinfo['Auto Update Mode'] = 'None'
    thisinfo['Current Version'] = ''
    thisinfo['Current Version Code'] = '0'
    thisinfo['Repo Type'] = ''
    thisinfo['Repo'] = ''
    thisinfo['Requires Root'] = False
    thisinfo['No Source Since'] = ''

    # General defaults...
    thisinfo['builds'] = []
    thisinfo['comments'] = []

    if metafile is None:
        return thisinfo

    mode = 0
    buildlines = []
    curcomments = []
    curbuild = None

    for line in metafile:
        line = line.rstrip('\r\n')
        if mode == 3:
            if not any(line.startswith(s) for s in (' ', '\t')):
                if 'commit' not in curbuild and 'disable' not in curbuild:
                    raise MetaDataException("No commit specified for {0} in {1}".format(
                        curbuild['version'], metafile.name))
                thisinfo['builds'].append(curbuild)
                add_comments('build:' + curbuild['version'])
                mode = 0
            else:
                if line.endswith('\\'):
                    buildlines.append(line[:-1].lstrip())
                else:
                    buildlines.append(line.lstrip())
                    bl = ''.join(buildlines)
                    bv = bl.split('=', 1)
                    if len(bv) != 2:
                        raise MetaDataException("Invalid build flag at {0} in {1}".
                                format(buildlines[0], metafile.name))
                    name, val = bv
                    if name in curbuild:
                        raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
                                format(name, curbuild['version'], metafile.name))
                    curbuild[name] = val.lstrip()
                    buildlines = []

        if mode == 0:
            if not line:
                continue
            if line.startswith("#"):
                curcomments.append(line)
                continue
            index = line.find(':')
            if index == -1:
                raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
            field = line[:index]
            value = line[index+1:]

            # Translate obsolete fields...
            if field == 'Market Version':
                field = 'Current Version'
            if field == 'Market Version Code':
                field = 'Current Version Code'

            fieldtype = metafieldtype(field)
            if fieldtype not in ['build', 'buildv2']:
                add_comments(field)
            if fieldtype == 'multiline':
                mode = 1
                thisinfo[field] = []
                if value:
                    raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
            elif fieldtype == 'string':
                if field == 'Category' and thisinfo['Categories'] == 'None':
                    thisinfo['Categories'] = value.replace(';',',')
                thisinfo[field] = value
            elif fieldtype == 'flag':
                if value == 'Yes':
                    thisinfo[field] = True
                elif value == 'No':
                    thisinfo[field] = False
                else:
                    raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name)
            elif fieldtype == 'build':
                if value.endswith("\\"):
                    mode = 2
                    buildlines = [value[:-1]]
                else:
                    thisinfo['builds'].append(parse_buildline([value]))
                    add_comments('build:' + thisinfo['builds'][-1]['version'])
            elif fieldtype == 'buildv2':
                curbuild = {}
                vv = value.split(',')
                if len(vv) != 2:
                    raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
                        format(value, metafile.name))
                curbuild['version'] = vv[0]
                curbuild['vercode'] = vv[1]
                try:
                    int(curbuild['vercode'])
                except:
                    raise MetaDataException("Invalid version code for build in " + metafile.name)
                buildlines = []
                mode = 3
            elif fieldtype == 'obsolete':
                pass        # Just throw it away!
            else:
                raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
        elif mode == 1:     # Multiline field
            if line == '.':
                mode = 0
            else:
                thisinfo[field].append(line)
        elif mode == 2:     # Line continuation mode in Build Version
            if line.endswith("\\"):
                buildlines.append(line[:-1])
            else:
                buildlines.append(line)
                thisinfo['builds'].append(
                    parse_buildline(buildlines))
                add_comments('build:' + thisinfo['builds'][-1]['version'])
                mode = 0
    add_comments(None)

    for key in bool_keys:
        for build in thisinfo['builds']:
            if key not in build:
                build[key] = False
                continue
            if build[key] == 'yes':
                build[key] = True
            elif build[key] == 'no':
                build[key] = False
            else:
                raise MetaDataException("Invalid value %s assigned to boolean build flag %s"
                        % (build[key], key))

    # Mode at end of file should always be 0...
    if mode == 1:
        raise MetaDataException(field + " not terminated in " + metafile.name)
    elif mode == 2:
        raise MetaDataException("Unterminated continuation in " + metafile.name)
    elif mode == 3:
        raise MetaDataException("Unterminated build in " + metafile.name)

    if not thisinfo['Description']:
        thisinfo['Description'].append('No description available')

    # Validate archive policy...
    if thisinfo['Archive Policy']:
        if not thisinfo['Archive Policy'].endswith(' versions'):
            raise MetaDataException("Invalid archive policy")
        try:
            versions = int(thisinfo['Archive Policy'][:-9])
            if versions < 1 or versions > 20:
                raise MetaDataException("Silly number of versions for archive policy")
        except:
            raise MetaDataException("Incomprehensible number of versions for archive policy")

    # Ensure all AntiFeatures are recognised...
    if thisinfo['AntiFeatures']:
        parts = thisinfo['AntiFeatures'].split(",")
        for part in parts:
            if (part != "Ads" and
                part != "Tracking" and
                part != "NonFreeNet" and
                part != "NonFreeDep" and
                part != "NonFreeAdd"):
                raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
                            + metafile.name)

    return thisinfo

def getvercode(build):
    return "%s" % (build['vercode'])

def getapkname(app, build):
    return "%s_%s.apk" % (app['id'], getvercode(build))

def getsrcname(app, build):
    return "%s_%s_src.tar.gz" % (app['id'], getvercode(build))

# Write a metadata file.
#
# 'dest'    - The path to the output file
# 'app'     - The app data
def write_metadata(dest, app):

    def writecomments(key):
        written = 0
        for pf, comment in app['comments']:
            if pf == key:
                mf.write(comment + '\n')
                written += 1
        if options.verbose and written > 0:
            print "...writing comments for " + (key if key else 'EOF')

    def writefield(field, value=None):
        writecomments(field)
        if value is None:
            value = app[field]
        mf.write(field + ':' + value + '\n')

    mf = open(dest, 'w')
    if app['Disabled']:
        writefield('Disabled')
    if app['AntiFeatures']:
        writefield('AntiFeatures')
    writefield('Categories')
    writefield('License')
    writefield('Web Site')
    writefield('Source Code')
    writefield('Issue Tracker')
    if app['Donate']:
        writefield('Donate')
    if app['FlattrID']:
        writefield('FlattrID')
    if app['Bitcoin']:
        writefield('Bitcoin')
    if app['Litecoin']:
        writefield('Litecoin')
    mf.write('\n')
    if app['Name']:
        writefield('Name')
    if app['Auto Name']:
        writefield('Auto Name')
    writefield('Summary')
    writefield('Description', '')
    for line in app['Description']:
        mf.write(line + '\n')
    mf.write('.\n')
    mf.write('\n')
    if app['Requires Root']:
        writefield('Requires Root', 'Yes')
        mf.write('\n')
    if app['Repo Type']:
        writefield('Repo Type')
        writefield('Repo')
        mf.write('\n')
    for build in app['builds']:
        writecomments('build:' + build['version'])
        mf.write('Build:')
        mf.write("%s,%s\n" % (
            build['version'],
            getvercode(build)))

        # This defines the preferred order for the build items - as in the
        # manual, they're roughly in order of application.
        keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
                    'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
                    'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
                    'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
                    'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
                    'preassemble', 'bindir', 'antcommand', 'novcheck']

        def write_builditem(key, value):
            if key not in ['version', 'vercode', 'origlines']:
                if key in bool_keys:
                    if not value:
                        return
                    value = 'yes'
                if options.verbose:
                    print "...writing {0} : {1}".format(key, value)
                outline = '    %s=' % key
                outline += '&& \\\n        '.join([s.lstrip() for s in value.split('&& ')])
                outline += '\n'
                mf.write(outline)

        for key in keyorder:
            if key in build:
                write_builditem(key, build[key])
        for key, value in build.iteritems():
            if not key in keyorder:
                write_builditem(key, value)
        mf.write('\n')

    if 'Maintainer Notes' in app:
        writefield('Maintainer Notes', '')
        for line in app['Maintainer Notes']:
            mf.write(line + '\n')
        mf.write('.\n')
        mf.write('\n')


    if app['Archive Policy']:
        writefield('Archive Policy')
    writefield('Auto Update Mode')
    writefield('Update Check Mode')
    if app['Vercode Operation']:
        writefield('Vercode Operation')
    if 'Update Check Data' in app:
        writefield('Update Check Data')
    if app['Current Version']:
        writefield('Current Version')
        writefield('Current Version Code')
    mf.write('\n')
    if app['No Source Since']:
        writefield('No Source Since')
        mf.write('\n')
    writecomments(None)
    mf.close()


# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
# returned by the parse_metadata function.
def read_metadata(xref=True, package=None):
    apps = []
    for basedir in ('metadata', 'tmp'):
        if not os.path.exists(basedir):
            os.makedirs(basedir)
    for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
        if package is None or metafile == os.path.join('metadata', package + '.txt'):
            try:
                appinfo = parse_metadata(metafile)
            except Exception, e:
                raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
            apps.append(appinfo)

    if xref:
        # Parse all descriptions at load time, just to ensure cross-referencing
        # errors are caught early rather than when they hit the build server.
        def linkres(link):
            for app in apps:
                if app['id'] == link:
                    return ("fdroid.app:" + link, "Dummy name - don't know yet")
            raise MetaDataException("Cannot resolve app id " + link)
        for app in apps:
            try:
                description_html(app['Description'], linkres)
            except Exception, e:
                raise MetaDataException("Problem with description of " + app['id'] +
                        " - " + str(e))

    return apps

# Formatter for descriptions. Create an instance, and call parseline() with
# each line of the description source from the metadata. At the end, call
# end() and then text_plain, text_wiki and text_html will contain the result.
class DescriptionFormatter:
    stNONE = 0
    stPARA = 1
    stUL = 2
    stOL = 3
    bold = False
    ital = False
    state = stNONE
    text_plain = ''
    text_wiki = ''
    text_html = ''
    linkResolver = None
    def __init__(self, linkres):
        self.linkResolver = linkres
    def endcur(self, notstates=None):
        if notstates and self.state in notstates:
            return
        if self.state == self.stPARA:
            self.endpara()
        elif self.state == self.stUL:
            self.endul()
        elif self.state == self.stOL:
            self.endol()
    def endpara(self):
        self.text_plain += '\n'
        self.text_html += '</p>'
        self.state = self.stNONE
    def endul(self):
        self.text_html += '</ul>'
        self.state = self.stNONE
    def endol(self):
        self.text_html += '</ol>'
        self.state = self.stNONE

    def formatted(self, txt, html):
        formatted = ''
        if html:
            txt = cgi.escape(txt)
        while True:
            index = txt.find("''")
            if index == -1:
                return formatted + txt
            formatted += txt[:index]
            txt = txt[index:]
            if txt.startswith("'''"):
                if html:
                    if self.bold:
                        formatted += '</b>'
                    else:
                        formatted += '<b>'
                self.bold = not self.bold
                txt = txt[3:]
            else:
                if html:
                    if self.ital:
                        formatted += '</i>'
                    else:
                        formatted += '<i>'
                self.ital = not self.ital
                txt = txt[2:]


    def linkify(self, txt):
        linkified_plain = ''
        linkified_html = ''
        while True:
            index = txt.find("[")
            if index == -1:
                return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
            linkified_plain += self.formatted(txt[:index], False)
            linkified_html += self.formatted(txt[:index], True)
            txt = txt[index:]
            if txt.startswith("[["):
                index = txt.find("]]")
                if index == -1:
                    raise MetaDataException("Unterminated ]]")
                url = txt[2:index]
                if self.linkResolver:
                    url, urltext = self.linkResolver(url)
                else:
                    urltext = url
                linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
                linkified_plain += urltext
                txt = txt[index+2:]
            else:
                index = txt.find("]")
                if index == -1:
                    raise MetaDataException("Unterminated ]")
                url = txt[1:index]
                index2 = url.find(' ')
                if index2 == -1:
                    urltxt = url
                else:
                    urltxt = url[index2 + 1:]
                    url = url[:index2]
                linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
                linkified_plain += urltxt
                if urltxt != url:
                    linkified_plain += ' (' + url + ')'
                txt = txt[index+1:]

    def addtext(self, txt):
        p, h = self.linkify(txt)
        self.text_plain += p
        self.text_html += h

    def parseline(self, line):
        self.text_wiki += line + '\n'
        if not line:
            self.endcur()
        elif line.startswith('*'):
            self.endcur([self.stUL])
            if self.state != self.stUL:
                self.text_html += '<ul>'
                self.state = self.stUL
            self.text_html += '<li>'
            self.text_plain += '*'
            self.addtext(line[1:])
            self.text_html += '</li>'
        elif line.startswith('#'):
            self.endcur([self.stOL])
            if self.state != self.stOL:
                self.text_html += '<ol>'
                self.state = self.stOL
            self.text_html += '<li>'
            self.text_plain += '*' #TODO: lazy - put the numbers in!
            self.addtext(line[1:])
            self.text_html += '</li>'
        else:
            self.endcur([self.stPARA])
            if self.state == self.stNONE:
                self.text_html += '<p>'
                self.state = self.stPARA
            elif self.state == self.stPARA:
                self.text_html += ' '
                self.text_plain += ' '
            self.addtext(line)

    def end(self):
        self.endcur()

# Parse multiple lines of description as written in a metadata file, returning
# a single string in plain text format.
def description_plain(lines, linkres):
    ps = DescriptionFormatter(linkres)
    for line in lines:
        ps.parseline(line)
    ps.end()
    return ps.text_plain

# Parse multiple lines of description as written in a metadata file, returning
# a single string in wiki format. Used for the Maintainer Notes field as well,
# because it's the same format.
def description_wiki(lines):
    ps = DescriptionFormatter(None)
    for line in lines:
        ps.parseline(line)
    ps.end()
    return ps.text_wiki

# Parse multiple lines of description as written in a metadata file, returning
# a single string in HTML format.
def description_html(lines,linkres):
    ps = DescriptionFormatter(linkres)
    for line in lines:
        ps.parseline(line)
    ps.end()
    return ps.text_html

def retrieve_string(xml_dir, string):
    if not string.startswith('@string/'):
        return string.replace("\\'","'")
    string_search = re.compile(r'.*"'+string[8:]+'".*>([^<]+?)<.*').search
    for xmlfile in glob.glob(os.path.join(xml_dir, '*.xml')):
        for line in file(xmlfile):
            matches = string_search(line)
            if matches:
                return retrieve_string(xml_dir, matches.group(1))
    return ''

# Return list of existing files that will be used to find the highest vercode
def manifest_paths(app_dir, flavour):

    possible_manifests = [ os.path.join(app_dir, 'AndroidManifest.xml'),
            os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
            os.path.join(app_dir, 'build.gradle') ]

    if flavour is not None:
        possible_manifests.append(
                os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
    
    return [path for path in possible_manifests if os.path.isfile(path)]

# Retrieve the package name
def fetch_real_name(app_dir, flavour):
    app_search = re.compile(r'.*<application.*').search
    name_search = re.compile(r'.*android:label="([^"]+)".*').search
    app_found = False
    for f in manifest_paths(app_dir, flavour):
        if not f.endswith(".xml"):
            continue
        xml_dir = os.path.join(f[:-19], 'res', 'values')
        for line in file(f):
            if not app_found:
                if app_search(line):
                    app_found = True
            if app_found:
                matches = name_search(line)
                if matches:
                    return retrieve_string(xml_dir, matches.group(1))
    return ''

# Retrieve the version name
def version_name(original, app_dir, flavour):
    for f in manifest_paths(app_dir, flavour):
        if not f.endswith(".xml"):
            continue
        xml_dir = os.path.join(f[:-19], 'res', 'values')
        string = retrieve_string(xml_dir, original)
        if string:
            return string
    return original

def ant_subprojects(root_dir):
    subprojects = []
    proppath = os.path.join(root_dir, 'project.properties')
    if not os.path.isfile(proppath):
        return subprojects
    with open(proppath) as f:
        for line in f.readlines():
            if not line.startswith('android.library.reference.'):
                continue
            path = line.split('=')[1].strip()
            relpath = os.path.join(root_dir, path)
            if not os.path.isdir(relpath):
                continue
            if options.verbose:
                print "Found subproject %s..." % path
            subprojects.append(path)
    return subprojects

# Extract some information from the AndroidManifest.xml at the given path.
# Returns (version, vercode, package), any or all of which might be None.
# All values returned are strings.
def parse_androidmanifests(paths):

    if not paths:
        return (None, None, None)

    vcsearch = re.compile(r'.*android:versionCode="([0-9]+?)".*').search
    vnsearch = re.compile(r'.*android:versionName="([^"]+?)".*').search
    psearch = re.compile(r'.*package="([^"]+)".*').search

    vcsearch_g = re.compile(r'.*versionCode[ =]*([0-9]+?)[^\d].*').search
    vnsearch_g = re.compile(r'.*versionName[ =]*"([^"]+?)".*').search
    psearch_g = re.compile(r'.*packageName[ =]*"([^"]+)".*').search

    max_version = None
    max_vercode = None
    max_package = None

    for path in paths:

        gradle = path.endswith("gradle")
        version = None
        vercode = None
        # Remember package name, may be defined separately from version+vercode
        package = max_package

        for line in file(path):
            if not package:
                if gradle:
                    matches = psearch_g(line)
                else:
                    matches = psearch(line)
                if matches:
                    package = matches.group(1)
            if not version:
                if gradle:
                    matches = vnsearch_g(line)
                else:
                    matches = vnsearch(line)
                if matches:
                    version = matches.group(1)
            if not vercode:
                if gradle:
                    matches = vcsearch_g(line)
                else:
                    matches = vcsearch(line)
                if matches:
                    vercode = matches.group(1)

        # Better some package name than nothing
        if max_package is None:
            max_package = package

        if max_vercode is None or (vercode is not None and vercode > max_vercode):
            max_version = version
            max_vercode = vercode
            max_package = package

    if max_version is None:
        max_version = "Unknown"

    return (max_version, max_vercode, max_package)

class BuildException(Exception):
    def __init__(self, value, stdout = None, stderr = None):
        self.value = value
        self.stdout = stdout
        self.stderr = stderr

    def get_wikitext(self):
        ret = repr(self.value) + "\n"
        if self.stdout:
            ret += "=stdout=\n"
            ret += "<pre>\n"
            ret += str(self.stdout)
            ret += "</pre>\n"
        if self.stderr:
            ret += "=stderr=\n"
            ret += "<pre>\n"
            ret += str(self.stderr)
            ret += "</pre>\n"
        return ret

    def __str__(self):
        ret = repr(self.value)
        if self.stdout:
            ret += "\n==== stdout begin ====\n%s\n==== stdout end ====" % self.stdout.strip()
        if self.stderr:
            ret += "\n==== stderr begin ====\n%s\n==== stderr end ====" % self.stderr.strip()
        return ret

class VCSException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class MetaDataException(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

def parse_srclib(metafile, **kw):

    thisinfo = {}
    if metafile and not isinstance(metafile, file):
        metafile = open(metafile, "r")

    # Defaults for fields that come from metadata
    thisinfo['Repo Type'] = ''
    thisinfo['Repo'] = ''
    thisinfo['Subdir'] = None
    thisinfo['Prepare'] = None
    thisinfo['Srclibs'] = None
    thisinfo['Update Project'] = None

    if metafile is None:
        return thisinfo

    for line in metafile:
        line = line.rstrip('\r\n')
        if not line or line.startswith("#"):
            continue

        index = line.find(':')
        if index == -1:
            raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
        field = line[:index]
        value = line[index+1:]

        if field == "Subdir":
            thisinfo[field] = value.split(',')
        else:
            thisinfo[field] = value

    return thisinfo

# Get the specified source library.
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, basepath=False,
        raw=False, prepare=True, preponly=False):

    if raw:
        name = spec
        ref = None
    else:
        name, ref = spec.split('@')

    srclib_path = os.path.join('srclibs', name + ".txt")

    if not os.path.exists(srclib_path):
        raise BuildException('srclib ' + name + ' not found.')

    srclib = parse_srclib(srclib_path)

    sdir = os.path.join(srclib_dir, name)

    if not preponly:
        vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
        vcs.srclib = (name, sdir)
        if ref:
            vcs.gotorevision(ref)

        if raw:
            return vcs

    libdir = None
    if subdir is not None:
        libdir = os.path.join(sdir, subdir)
    elif srclib["Subdir"] is not None:
        for subdir in srclib["Subdir"]:
            libdir_candidate = os.path.join(sdir, subdir)
            if os.path.exists(libdir_candidate):
                libdir = libdir_candidate
                break

    if libdir is None:
        libdir = sdir

    if srclib["Srclibs"]:
        n=1
        for lib in srclib["Srclibs"].split(','):
            s_tuple = None
            for t in srclibpaths:
                if t[0] == lib:
                    s_tuple = t
                    break
            if s_tuple is None:
                raise BuildException('Missing recursive srclib %s for %s' % (
                    lib, name))
            place_srclib(libdir, n, s_tuple[2])
            n+=1

    if prepare:

        if srclib["Prepare"] is not None:
            cmd = replace_config_vars(srclib["Prepare"])

            p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
            if p.returncode != 0:
                raise BuildException("Error running prepare command for srclib %s"
                        % name, p.stdout, p.stderr)
        
        if srclib["Update Project"] == "Yes":
            print "Updating srclib %s at path %s" % (name, libdir)
            p = FDroidPopen([
                os.path.join(config['sdk_path'], 'tools', 'android'),
                'update', 'project', '-p', libdir])
            # Check to see whether an error was returned without a proper exit
            # code (this is the case for the 'no target set or target invalid'
            # error)
            if p.returncode != 0 or (p.stderr != "" and
                    p.stderr.startswith("Error: ")):
                raise BuildException("Failed to update srclib project {0}"
                        .format(name), p.stdout, p.stderr)

            remove_signing_keys(libdir)

    if basepath:
        return sdir
    return libdir


# Prepare the source code for a particular build
#  'vcs'         - the appropriate vcs object for the application
#  'app'         - the application details from the metadata
#  'build'       - the build details from the metadata
#  'build_dir'   - the path to the build directory, usually
#                   'build/app.id'
#  'srclib_dir'  - the path to the source libraries directory, usually
#                   'build/srclib'
#  'extlib_dir'  - the path to the external libraries directory, usually
#                   'build/extlib'
# Returns the (root, srclibpaths) where:
#   'root' is the root directory, which may be the same as 'build_dir' or may
#          be a subdirectory of it.
#   'srclibpaths' is information on the srclibs being used
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):

    # Optionally, the actual app source can be in a subdirectory...
    if 'subdir' in build:
        root_dir = os.path.join(build_dir, build['subdir'])
    else:
        root_dir = build_dir

    # Get a working copy of the right revision...
    print "Getting source for revision " + build['commit']
    vcs.gotorevision(build['commit'])

    # Check that a subdir (if we're using one) exists. This has to happen
    # after the checkout, since it might not exist elsewhere...
    if not os.path.exists(root_dir):
        raise BuildException('Missing subdir ' + root_dir)

    # Initialise submodules if requred...
    if build['submodules']:
        if options.verbose:
            print "Initialising submodules..."
        vcs.initsubmodules()

    # Run an init command if one is required...
    if 'init' in build:
        cmd = replace_config_vars(build['init'])
        if options.verbose:
            print "Running 'init' commands in %s" % root_dir

        p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
        if p.returncode != 0:
            raise BuildException("Error running init command for %s:%s" %
                    (app['id'], build['version']), p.stdout, p.stderr)

    # Generate (or update) the ant build file, build.xml...
    updatemode = build.get('update', 'auto')
    if (updatemode != 'no'
            and build.get('maven', 'no') == 'no'
            and build.get('gradle', 'no') == 'no'):
        parms = [os.path.join(config['sdk_path'], 'tools', 'android'),
                'update', 'project']
        if 'target' in build and build['target']:
            parms += ['-t', build['target']]
        update_dirs = None
        if updatemode == 'auto':
            update_dirs = ['.'] + ant_subprojects(root_dir)
        else:
            update_dirs = [d.strip() for d in updatemode.split(';')]
        # Force build.xml update if necessary...
        if updatemode == 'force' or 'target' in build:
            if updatemode == 'force':
                update_dirs = ['.']
            buildxml = os.path.join(root_dir, 'build.xml')
            if os.path.exists(buildxml):
                print 'Force-removing old build.xml'
                os.remove(buildxml)

        for d in update_dirs:
            # Remove gen and bin dirs in libraries
            # rid of them...
            for baddir in [
                    'gen', 'bin', 'obj', # ant
                    'libs/armeabi-v7a', 'libs/armeabi', # jni
                    'libs/mips', 'libs/x86']:
                badpath = os.path.join(root_dir, d, baddir)
                if os.path.exists(badpath):
                    print "Removing '%s'" % badpath
                    shutil.rmtree(badpath)
            dparms = parms + ['-p', d]
            if options.verbose:
                if d == '.':
                    print "Updating main project..."
                else:
                    print "Updating subproject %s..." % d
            p = FDroidPopen(dparms, cwd=root_dir)
            # Check to see whether an error was returned without a proper exit
            # code (this is the case for the 'no target set or target invalid'
            # error)
            if p.returncode != 0 or (p.stderr != "" and
                    p.stderr.startswith("Error: ")):
                raise BuildException("Failed to update project at %s" % d,
                        p.stdout, p.stderr)

    remove_signing_keys(build_dir)

    # Update the local.properties file...
    localprops = [ os.path.join(build_dir, 'local.properties') ]
    if 'subdir' in build:
        localprops += [ os.path.join(root_dir, 'local.properties') ]
    for path in localprops:
        if not os.path.isfile(path):
            continue
        if options.verbose:
            print "Updating properties file at %s" % path
        f = open(path, 'r')
        props = f.read()
        f.close()
        props += '\n'
        # Fix old-fashioned 'sdk-location' by copying
        # from sdk.dir, if necessary...
        if build['oldsdkloc']:
            sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
                re.S|re.M).group(1)
            props += "sdk-location=%s\n" % sdkloc
        else:
            props += "sdk.dir=%s\n" % config['sdk_path']
            props += "sdk-location=%s\n" % ['sdk_path']
        # Add ndk location...
        props += "ndk.dir=%s\n" % config['ndk_path']
        props += "ndk-location=%s\n" % config['ndk_path']
        # Add java.encoding if necessary...
        if 'encoding' in build:
            props += "java.encoding=%s\n" % build['encoding']
        f = open(path, 'w')
        f.write(props)
        f.close()

    flavour = None
    if build.get('gradle', 'no') != 'no':
        flavour = build['gradle'].split('@')[0]
        if flavour in ['main', 'yes', '']:
            flavour = None

    # Remove forced debuggable flags
    print "Removing debuggable flags..."
    for path in manifest_paths(root_dir, flavour):
        if not os.path.isfile(path):
            continue
        if subprocess.call(['sed','-i',
            's/android:debuggable="[^"]*"//g', path]) != 0:
            raise BuildException("Failed to remove debuggable flags")

    # Insert version code and number into the manifest if necessary...
    if build['forceversion']:
        print "Changing the version name..."
        for path in manifest_paths(root_dir, flavour):
            if not os.path.isfile(path):
                continue
            if path.endswith('.xml'):
                if subprocess.call(['sed','-i',
                    's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
                    path]) != 0:
                    raise BuildException("Failed to amend manifest")
            elif path.endswith('.gradle'):
                if subprocess.call(['sed','-i',
                    's/versionName[ ]*=[ ]*"[^"]*"/versionName = "' + build['version'] + '"/g',
                    path]) != 0:
                    raise BuildException("Failed to amend build.gradle")
    if build['forcevercode']:
        print "Changing the version code..."
        for path in manifest_paths(root_dir, flavour):
            if not os.path.isfile(path):
                continue
            if path.endswith('.xml'):
                if subprocess.call(['sed','-i',
                    's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
                    path]) != 0:
                    raise BuildException("Failed to amend manifest")
            elif path.endswith('.gradle'):
                if subprocess.call(['sed','-i',
                    's/versionCode[ ]*=[ ]*[0-9]*/versionCode = ' + build['vercode'] + '/g',
                    path]) != 0:
                    raise BuildException("Failed to amend build.gradle")

    # Delete unwanted files...
    if 'rm' in build:
        for part in build['rm'].split(';'):
            dest = os.path.join(build_dir, part.strip())
            rdest = os.path.realpath(dest)
            if not rdest.startswith(os.path.realpath(build_dir)):
                raise BuildException("rm for {0} is outside build root {1}".format(
                    os.path.realpath(build_dir),os.path.realpath(dest)))
            if rdest == os.path.realpath(build_dir):
                raise BuildException("rm removes whole build directory")
            if os.path.exists(rdest):
                subprocess.call('rm -rf ' + rdest, shell=True)

    # Fix apostrophes translation files if necessary...
    if build['fixapos']:
        for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
            for filename in files:
                if filename.endswith('.xml'):
                    if subprocess.call(['sed','-i','s@' +
                        r"\([^\\]\)'@\1\\'" +
                        '@g',
                        os.path.join(root, filename)]) != 0:
                        raise BuildException("Failed to amend " + filename)

    # Fix translation files if necessary...
    if build['fixtrans']:
        for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
            for filename in files:
                if filename.endswith('.xml'):
                    f = open(os.path.join(root, filename))
                    changed = False
                    outlines = []
                    for line in f:
                        num = 1
                        index = 0
                        oldline = line
                        while True:
                            index = line.find("%", index)
                            if index == -1:
                                break
                            next = line[index+1:index+2]
                            if next == "s" or next == "d":
                                line = (line[:index+1] +
                                        str(num) + "$" +
                                        line[index+1:])
                                num += 1
                                index += 3
                            else:
                                index += 1
                        # We only want to insert the positional arguments
                        # when there is more than one argument...
                        if oldline != line:
                            if num > 2:
                                changed = True
                            else:
                                line = oldline
                        outlines.append(line)
                    f.close()
                    if changed:
                        f = open(os.path.join(root, filename), 'w')
                        f.writelines(outlines)
                        f.close()

    # Add required external libraries...
    if 'extlibs' in build:
        print "Collecting prebuilt libraries..."
        libsdir = os.path.join(root_dir, 'libs')
        if not os.path.exists(libsdir):
            os.mkdir(libsdir)
        for lib in build['extlibs'].split(';'):
            lib = lib.strip()
            if options.verbose:
                print "...installing extlib {0}".format(lib)
            libf = os.path.basename(lib)
            libsrc = os.path.join(extlib_dir, lib)
            if not os.path.exists(libsrc):
                raise BuildException("Missing extlib file {0}".format(libsrc))
            shutil.copyfile(libsrc, os.path.join(libsdir, libf))

    # Get required source libraries...
    srclibpaths = []
    if 'srclibs' in build:
        print "Collecting source libraries..."
        for lib in build['srclibs'].split(';'):
            number = None
            subdir = None
            lib = lib.strip()
            name, ref = lib.split('@')
            if ':' in name:
                number, name = name.split(':', 1)
            if '/' in name:
                name, subdir = name.split('/',1)
            libpath = getsrclib(name+'@'+ref, srclib_dir, srclibpaths, subdir, preponly=onserver)
            srclibpaths.append((name, number, libpath))
            place_srclib(root_dir, int(number) if number else None, libpath)
                
    basesrclib = vcs.getsrclib()
    # If one was used for the main source, add that too.
    if basesrclib:
        srclibpaths.append(basesrclib)

    # Apply patches if any
    if 'patch' in build:
        for patch in build['patch'].split(';'):
            patch = patch.strip()
            print "Applying " + patch
            patch_path = os.path.join('metadata', app['id'], patch)
            if subprocess.call(['patch', '-p1',
                            '-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
                raise BuildException("Failed to apply patch %s" % patch_path)

    # Run a pre-build command if one is required...
    if 'prebuild' in build:
        cmd = replace_config_vars(build['prebuild'])

        # Substitute source library paths into prebuild commands...
        for name, number, libpath in srclibpaths:
            libpath = os.path.relpath(libpath, root_dir)
            cmd = cmd.replace('$$' + name + '$$', libpath)

        if options.verbose:
            print "Running 'prebuild' commands in %s" % root_dir

        p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
        if p.returncode != 0:
            raise BuildException("Error running prebuild command for %s:%s" %
                    (app['id'], build['version']), p.stdout, p.stderr)

    return (root_dir, srclibpaths)


# Scan the source code in the given directory (and all subdirectories)
# and return a list of potential problems.
def scan_source(build_dir, root_dir, thisbuild):

    problems = []

    # Common known non-free blobs (always lower case):
    usual_suspects = ['flurryagent',
                      'paypal_mpl',
                      'libgoogleanalytics',
                      'admob-sdk-android',
                      'googleadview',
                      'googleadmobadssdk',
                      'google-play-services',
                      'crittercism',
                      'heyzap',
                      'jpct-ae',
                      'youtubeandroidplayerapi',
                      'bugsense',
                      'crashlytics']

    def getpaths(field):
        paths = []
        if field not in thisbuild:
            return paths
        for p in thisbuild[field].split(';'):
            p = p.strip()
            if p == '.':
                p = '/'
            elif p.startswith('./'):
                p = p[1:]
            elif not p.startswith('/'):
                p = '/' + p;
            if p not in paths:
                paths.append(p)
        return paths

    scanignore = getpaths('scanignore')
    scandelete = getpaths('scandelete')

    ms = magic.open(magic.MIME_TYPE)
    ms.load()

    def toignore(fd):
        for i in scanignore:
            if fd.startswith(i):
                return True
        return False

    def todelete(fd):
        for i in scandelete:
            if fd.startswith(i):
                return True
        return False

    def removeproblem(what, fd, fp):
        print 'Removing %s at %s' % (what, fd)
        os.remove(fp)
    
    def handleproblem(what, fd, fp):
        if todelete(fd):
            removeproblem(what, fd, fp)
        else:
            problems.append('Found %s at %s' % (what, fd))

    # Iterate through all files in the source code...
    for r,d,f in os.walk(build_dir):
        for curfile in f:

            if '/.hg' in r or '/.git' in r or '/.svn' in r:
                continue

            # Path (relative) to the file...
            fp = os.path.join(r, curfile)
            fd = fp[len(build_dir):]

            # Check if this file has been explicitly excluded from scanning...
            if toignore(fd):
                continue

            for suspect in usual_suspects:
                if suspect in curfile.lower():
                    handleproblem('usual supect', fd, fp)

            mime = ms.file(fp)
            if mime == 'application/x-sharedlib':
                handleproblem('shared library', fd, fp)
            elif mime == 'application/x-archive':
                handleproblem('static library', fd, fp)
            elif mime == 'application/x-executable':
                handleproblem('binary executable', fd, fp)
            elif mime == 'application/jar' and fp.endswith('.apk'):
                removeproblem('APK file', fd, fp)

            elif curfile.endswith('.java'):
                for line in file(fp):
                    if 'DexClassLoader' in line:
                        handleproblem('DexClassLoader', fd, fp)
                        break
    ms.close()

    # Presence of a jni directory without buildjni=yes might
    # indicate a problem... (if it's not a problem, explicitly use
    # buildjni=no to bypass this check)
    if (os.path.exists(os.path.join(root_dir, 'jni')) and 
            thisbuild.get('buildjni') is None):
        msg = 'Found jni directory, but buildjni is not enabled'
        problems.append(msg)

    return problems


class KnownApks:

    def __init__(self):
        self.path = os.path.join('stats', 'known_apks.txt')
        self.apks = {}
        if os.path.exists(self.path):
            for line in file( self.path):
                t = line.rstrip().split(' ')
                if len(t) == 2:
                    self.apks[t[0]] = (t[1], None)
                else:
                    self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
        self.changed = False

    def writeifchanged(self):
        if self.changed:
            if not os.path.exists('stats'):
                os.mkdir('stats')
            f = open(self.path, 'w')
            lst = []
            for apk, app in self.apks.iteritems():
                appid, added = app
                line = apk + ' ' + appid
                if added:
                    line += ' ' + time.strftime('%Y-%m-%d', added)
                lst.append(line)
            for line in sorted(lst):
                f.write(line + '\n')
            f.close()

    # Record an apk (if it's new, otherwise does nothing)
    # Returns the date it was added.
    def recordapk(self, apk, app):
        if not apk in self.apks:
            self.apks[apk] = (app, time.gmtime(time.time()))
            self.changed = True
        _, added = self.apks[apk]
        return added

    # Look up information - given the 'apkname', returns (app id, date added/None).
    # Or returns None for an unknown apk.
    def getapp(self, apkname):
        if apkname in self.apks:
            return self.apks[apkname]
        return None

    # Get the most recent 'num' apps added to the repo, as a list of package ids
    # with the most recent first.
    def getlatest(self, num):
        apps = {}
        for apk, app in self.apks.iteritems():
            appid, added = app
            if added:
                if appid in apps:
                    if apps[appid] > added:
                        apps[appid] = added
                else:
                    apps[appid] = added
        sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
        lst = [app for app,added in sortedapps]
        lst.reverse()
        return lst

def isApkDebuggable(apkfile, config):
    """Returns True if the given apk file is debuggable

    :param apkfile: full path to the apk to check"""

    p = subprocess.Popen([os.path.join(config['sdk_path'],
        'build-tools', config['build_tools'], 'aapt'),
        'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
        stdout=subprocess.PIPE)
    output = p.communicate()[0]
    if p.returncode != 0:
        print "ERROR: Failed to get apk manifest information"
        sys.exit(1)
    for line in output.splitlines():
        if line.find('android:debuggable') != -1 and not line.endswith('0x0'):
            return True
    return False


class AsynchronousFileReader(threading.Thread):
    '''
    Helper class to implement asynchronous reading of a file
    in a separate thread. Pushes read lines on a queue to
    be consumed in another thread.
    '''
 
    def __init__(self, fd, queue):
        assert isinstance(queue, Queue.Queue)
        assert callable(fd.readline)
        threading.Thread.__init__(self)
        self._fd = fd
        self._queue = queue
 
    def run(self):
        '''The body of the tread: read lines and put them on the queue.'''
        for line in iter(self._fd.readline, ''):
            self._queue.put(line)
 
    def eof(self):
        '''Check whether there is no more content to expect.'''
        return not self.is_alive() and self._queue.empty()

class PopenResult:
    returncode = None
    stdout = ''
    stderr = ''
    stdout_apk = ''

def FDroidPopen(commands, cwd=None):
    """
    Runs a command the FDroid way and returns return code and output

    :param commands, cwd: like subprocess.Popen
    """

    if options.verbose:
        if cwd is not None:
            print "Directory: %s" % cwd
        print " > %s" % ' '.join(commands)

    result = PopenResult()
    p = subprocess.Popen(commands, cwd=cwd,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    
    stdout_queue = Queue.Queue()
    stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
    stdout_reader.start()
    stderr_queue = Queue.Queue()
    stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
    stderr_reader.start()
    
    # Check the queues for output (until there is no more to get)
    while not stdout_reader.eof() or not stderr_reader.eof():
        # Show what we received from standard output
        while not stdout_queue.empty():
            line = stdout_queue.get()
            if options.verbose:
                # Output directly to console
                sys.stdout.write(line)
                sys.stdout.flush()
            result.stdout += line

        # Show what we received from standard error
        while not stderr_queue.empty():
            line = stderr_queue.get()
            if options.verbose:
                # Output directly to console
                sys.stderr.write(line)
                sys.stderr.flush()
            result.stderr += line
        time.sleep(0.2)

    p.communicate()
    result.returncode = p.returncode
    return result

def remove_signing_keys(build_dir):
    for root, dirs, files in os.walk(build_dir):
        if 'build.gradle' in files:
            path = os.path.join(root, 'build.gradle')
            changed = False

            if options.verbose:
                print "Cleaned build.gradle of keysigning configs at %s" % path

            with open(path, "r") as o:
                lines = o.readlines()
            
            opened = 0
            with open(path, "w") as o:
                for line in lines:
                    if 'signingConfigs ' in line:
                        opened = 1
                        changed = True
                    elif opened > 0:
                        if '{' in line:
                            opened += 1
                        elif '}' in line:
                            opened -=1
                    elif any(s in line for s in (
                            ' signingConfig ',
                            'android.signingConfigs.',
                            'variant.outputFile = ',
                            '.readLine(')):
                        changed = True
                    else:
                        o.write(line)

        for propfile in ('build.properties', 'default.properties', 'ant.properties'):
            if propfile in files:
                path = os.path.join(root, propfile)
                changed = False

                with open(path, "r") as o:
                    lines = o.readlines()

                with open(path, "w") as o:
                    for line in lines:
                        if line.startswith('key.store'):
                            changed = True
                        else:
                            o.write(line)

                if changed and options.verbose:
                    print "Cleaned %s of keysigning configs at %s" % (propfile,path)

def replace_config_vars(cmd):
    cmd = cmd.replace('$$SDK$$', config['sdk_path'])
    cmd = cmd.replace('$$NDK$$', config['ndk_path'])
    cmd = cmd.replace('$$MVN3$$', config['mvn3'])
    return cmd

def place_srclib(root_dir, number, libpath):
    if not number:
        return
    relpath = os.path.relpath(libpath, root_dir)
    proppath = os.path.join(root_dir, 'project.properties')

    with open(proppath, "r") as o:
        lines = o.readlines()

    with open(proppath, "w") as o:
        placed = False
        for line in lines:
            if line.startswith('android.library.reference.%d=' % number):
                o.write('android.library.reference.%d=%s\n' % (number,relpath))
                placed = True
            else:
                o.write(line)
        if not placed:
            o.write('android.library.reference.%d=%s\n' % (number,relpath))