mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-14 19:10:11 +01:00
1644 lines
61 KiB
Python
1644 lines
61 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# common.py - part of the FDroid server tools
|
|
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import glob, os, sys, re
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
import operator
|
|
import cgi
|
|
|
|
def getvcs(vcstype, remote, local, sdk_path):
|
|
if vcstype == 'git':
|
|
return vcs_git(remote, local, sdk_path)
|
|
elif vcstype == 'svn':
|
|
return vcs_svn(remote, local, sdk_path)
|
|
elif vcstype == 'git-svn':
|
|
return vcs_gitsvn(remote, local, sdk_path)
|
|
elif vcstype == 'hg':
|
|
return vcs_hg(remote, local, sdk_path)
|
|
elif vcstype == 'bzr':
|
|
return vcs_bzr(remote, local, sdk_path)
|
|
elif vcstype == 'srclib':
|
|
return vcs_srclib(remote, local, sdk_path)
|
|
raise VCSException("Invalid vcs type " + vcstype)
|
|
|
|
class vcs:
|
|
def __init__(self, remote, local, sdk_path):
|
|
|
|
self.sdk_path = sdk_path
|
|
|
|
# It's possible to sneak a username and password in with
|
|
# the remote address... (this really only applies to svn
|
|
# and we should probably be more specific!)
|
|
index = remote.find('@')
|
|
if index != -1:
|
|
self.username = remote[:index]
|
|
remote = remote[index+1:]
|
|
index = self.username.find(':')
|
|
if index == -1:
|
|
raise VCSException("Password required with username")
|
|
self.password = self.username[index+1:]
|
|
self.username = self.username[:index]
|
|
else:
|
|
self.username = None
|
|
|
|
self.remote = remote
|
|
self.local = local
|
|
self.refreshed = False
|
|
self.srclib = None
|
|
|
|
# Take the local repository to a clean version of the given revision, which
|
|
# is specificed in the VCS's native format. Beforehand, the repository can
|
|
# be dirty, or even non-existent. If the repository does already exist
|
|
# locally, it will be updated from the origin, but only once in the
|
|
# lifetime of the vcs object.
|
|
# None is acceptable for 'rev' if you know you are cloning a clean copy of
|
|
# the repo - otherwise it must specify a valid revision.
|
|
def gotorevision(self, rev):
|
|
|
|
# The .fdroidvcs-id file for a repo tells us what VCS type
|
|
# and remote that directory was created from, allowing us to drop it
|
|
# automatically if either of those things changes.
|
|
fdpath = os.path.join(self.local, '..',
|
|
'.fdroidvcs-' + os.path.basename(self.local))
|
|
cdata = self.repotype() + ' ' + self.remote
|
|
writeback = True
|
|
if os.path.exists(self.local):
|
|
if os.path.exists(fdpath):
|
|
with open(fdpath, 'r') as f:
|
|
fsdata = f.read()
|
|
if fsdata == cdata:
|
|
writeback = False
|
|
else:
|
|
print "*** Repository details changed - deleting ***"
|
|
shutil.rmtree(self.local)
|
|
|
|
self.gotorevisionx(rev)
|
|
|
|
# If necessary, write the .fdroidvcs file.
|
|
if writeback:
|
|
with open(fdpath, 'w') as f:
|
|
f.write(cdata)
|
|
|
|
# Derived classes need to implement this. It's called once basic checking
|
|
# has been performend.
|
|
def gotorevisionx(self, rev):
|
|
raise VCSException("This VCS type doesn't define gotorevisionx")
|
|
|
|
# Initialise and update submodules
|
|
def initsubmodules(self):
|
|
raise VCSException('Submodules not supported for this vcs type')
|
|
|
|
# Get a list of all known tags
|
|
def gettags(self):
|
|
raise VCSException('gettags not supported for this vcs type')
|
|
|
|
# Returns the srclib (name, path) used in setting up the current
|
|
# revision, or None.
|
|
def getsrclib(self):
|
|
return self.srclib
|
|
|
|
class vcs_git(vcs):
|
|
|
|
def repotype(self):
|
|
return 'git'
|
|
|
|
# If the local directory exists, but is somehow not a git repository, git
|
|
# will traverse up the directory tree until it finds one that is (i.e.
|
|
# fdroidserver) and then we'll proceed to destroy it! This is called as
|
|
# a safety check.
|
|
def checkrepo(self):
|
|
p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
|
|
stdout=subprocess.PIPE, cwd=self.local)
|
|
result = p.communicate()[0].rstrip()
|
|
if not result.endswith(self.local):
|
|
raise VCSException('Repository mismatch')
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
# Brand new checkout...
|
|
if subprocess.call(['git', 'clone', self.remote, self.local]) != 0:
|
|
raise VCSException("Git clone failed")
|
|
self.checkrepo()
|
|
else:
|
|
self.checkrepo()
|
|
# Discard any working tree changes...
|
|
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
|
|
raise VCSException("Git reset failed")
|
|
# Remove untracked files now, in case they're tracked in the target
|
|
# revision (it happens!)...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
if not self.refreshed:
|
|
# Get latest commits and tags from remote...
|
|
if subprocess.call(['git', 'fetch', 'origin'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git fetch failed")
|
|
if subprocess.call(['git', 'fetch', '--tags', 'origin'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git fetch failed")
|
|
self.refreshed = True
|
|
# Check out the appropriate revision...
|
|
if rev:
|
|
if subprocess.call(['git', 'checkout', rev], cwd=self.local) != 0:
|
|
raise VCSException("Git checkout failed")
|
|
# Get rid of any uncontrolled files left behind...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
|
|
def initsubmodules(self):
|
|
self.checkrepo()
|
|
if subprocess.call(['git', 'submodule', 'init'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git submodule init failed")
|
|
if subprocess.call(['git', 'submodule', 'update'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git submodule update failed")
|
|
|
|
def gettags(self):
|
|
self.checkrepo()
|
|
p = subprocess.Popen(['git', 'tag'],
|
|
stdout=subprocess.PIPE, cwd=self.local)
|
|
return p.communicate()[0].splitlines()
|
|
|
|
|
|
class vcs_gitsvn(vcs):
|
|
|
|
def repotype(self):
|
|
return 'git-svn'
|
|
|
|
# If the local directory exists, but is somehow not a git repository, git
|
|
# will traverse up the directory tree until it finds one that is (i.e.
|
|
# fdroidserver) and then we'll proceed to destory it! This is called as
|
|
# a safety check.
|
|
def checkrepo(self):
|
|
p = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
|
|
stdout=subprocess.PIPE, cwd=self.local)
|
|
result = p.communicate()[0].rstrip()
|
|
if not result.endswith(self.local):
|
|
raise VCSException('Repository mismatch')
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
# Brand new checkout...
|
|
if subprocess.call(['git', 'svn', 'clone', self.remote, self.local]) != 0:
|
|
raise VCSException("Git clone failed")
|
|
self.checkrepo()
|
|
else:
|
|
self.checkrepo()
|
|
# Discard any working tree changes...
|
|
if subprocess.call(['git', 'reset', '--hard'], cwd=self.local) != 0:
|
|
raise VCSException("Git reset failed")
|
|
# Remove untracked files now, in case they're tracked in the target
|
|
# revision (it happens!)...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
if not self.refreshed:
|
|
# Get new commits and tags from repo...
|
|
if subprocess.call(['git', 'svn', 'rebase'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Git svn rebase failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
# Figure out the git commit id corresponding to the svn revision...
|
|
p = subprocess.Popen(['git', 'svn', 'find-rev', 'r' + rev],
|
|
cwd=self.local, stdout=subprocess.PIPE)
|
|
rev = p.communicate()[0].rstrip()
|
|
if p.returncode != 0 or len(rev) == 0:
|
|
raise VCSException("Failed to get git treeish from svn rev")
|
|
# Check out the appropriate revision...
|
|
if subprocess.call(['git', 'checkout', rev], cwd=self.local) != 0:
|
|
raise VCSException("Git checkout failed")
|
|
# Get rid of any uncontrolled files left behind...
|
|
if subprocess.call(['git', 'clean', '-dffx'], cwd=self.local) != 0:
|
|
raise VCSException("Git clean failed")
|
|
|
|
class vcs_svn(vcs):
|
|
|
|
def repotype(self):
|
|
return 'svn'
|
|
|
|
def userargs(self):
|
|
if self.username is None:
|
|
return ['--non-interactive']
|
|
return ['--username', self.username,
|
|
'--password', self.password,
|
|
'--non-interactive']
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
if subprocess.call(['svn', 'checkout', self.remote, self.local] +
|
|
self.userargs()) != 0:
|
|
raise VCSException("Svn checkout failed")
|
|
else:
|
|
for svncommand in (
|
|
'svn revert -R .',
|
|
r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
|
|
if subprocess.call(svncommand, cwd=self.local,
|
|
shell=True) != 0:
|
|
raise VCSException("Svn reset failed")
|
|
if not self.refreshed:
|
|
if subprocess.call(['svn', 'update'] +
|
|
self.userargs(), cwd=self.local) != 0:
|
|
raise VCSException("Svn update failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
revargs = ['-r', rev]
|
|
if subprocess.call(['svn', 'update', '--force'] + revargs +
|
|
self.userargs(), cwd=self.local) != 0:
|
|
raise VCSException("Svn update failed")
|
|
|
|
|
|
class vcs_hg(vcs):
|
|
|
|
def repotype(self):
|
|
return 'hg'
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
if subprocess.call(['hg', 'clone', self.remote, self.local]) !=0:
|
|
raise VCSException("Hg clone failed")
|
|
else:
|
|
if subprocess.call('hg status -u | xargs rm -rf',
|
|
cwd=self.local, shell=True) != 0:
|
|
raise VCSException("Hg clean failed")
|
|
if not self.refreshed:
|
|
if subprocess.call(['hg', 'pull'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Hg pull failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
revargs = [rev]
|
|
if subprocess.call(['hg', 'checkout', '-C'] + revargs,
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Hg checkout failed")
|
|
|
|
|
|
class vcs_bzr(vcs):
|
|
|
|
def repotype(self):
|
|
return 'bzr'
|
|
|
|
def gotorevisionx(self, rev):
|
|
if not os.path.exists(self.local):
|
|
if subprocess.call(['bzr', 'branch', self.remote, self.local]) != 0:
|
|
raise VCSException("Bzr branch failed")
|
|
else:
|
|
if subprocess.call(['bzr', 'clean-tree', '--force',
|
|
'--unknown', '--ignored'], cwd=self.local) != 0:
|
|
raise VCSException("Bzr revert failed")
|
|
if not self.refreshed:
|
|
if subprocess.call(['bzr', 'pull'],
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Bzr update failed")
|
|
self.refreshed = True
|
|
if rev:
|
|
revargs = ['-r', rev]
|
|
if subprocess.call(['bzr', 'revert'] + revargs,
|
|
cwd=self.local) != 0:
|
|
raise VCSException("Bzr revert failed")
|
|
|
|
class vcs_srclib(vcs):
|
|
|
|
def repotype(self):
|
|
return 'srclib'
|
|
|
|
def gotorevisionx(self, rev):
|
|
|
|
# Yuk...
|
|
extlib_dir = 'build/extlib'
|
|
|
|
if os.path.exists(self.local):
|
|
shutil.rmtree(self.local)
|
|
|
|
if self.remote.find(':') != -1:
|
|
srclib, path = self.remote.split(':')
|
|
else:
|
|
srclib = self.remote
|
|
path = None
|
|
libdir = getsrclib(srclib + '@' + rev, extlib_dir, self.sdk_path)
|
|
self.srclib = (srclib, libdir)
|
|
if path:
|
|
libdir = os.path.join(libdir, path)
|
|
shutil.copytree(libdir, self.local)
|
|
return self.local
|
|
|
|
|
|
# Get the type expected for a given metadata field.
|
|
def metafieldtype(name):
|
|
if name == 'Description':
|
|
return 'multiline'
|
|
if name == 'Requires Root':
|
|
return 'flag'
|
|
if name == 'Build Version':
|
|
return 'build'
|
|
if name == 'Use Built':
|
|
return 'obsolete'
|
|
return 'string'
|
|
|
|
|
|
# Parse metadata for a single application.
|
|
#
|
|
# 'metafile' - the filename to read. The package id for the application comes
|
|
# from this filename. Pass None to get a blank entry.
|
|
#
|
|
# Returns a dictionary containing all the details of the application. There are
|
|
# two major kinds of information in the dictionary. Keys beginning with capital
|
|
# letters correspond directory to identically named keys in the metadata file.
|
|
# Keys beginning with lower case letters are generated in one way or another,
|
|
# and are not found verbatim in the metadata.
|
|
#
|
|
# Known keys not originating from the metadata are:
|
|
#
|
|
# 'id' - the application's package ID
|
|
# 'builds' - a list of dictionaries containing build information
|
|
# for each defined build
|
|
# 'comments' - a list of comments from the metadata file. Each is
|
|
# a tuple of the form (field, comment) where field is
|
|
# the name of the field it preceded in the metadata
|
|
# file. Where field is None, the comment goes at the
|
|
# end of the file. Alternatively, 'build:version' is
|
|
# for a comment before a particular build version.
|
|
# 'descriptionlines' - original lines of description as formatted in the
|
|
# metadata file.
|
|
#
|
|
def parse_metadata(metafile, **kw):
|
|
|
|
def parse_buildline(lines):
|
|
value = "".join(lines)
|
|
parts = [p.replace("\\,", ",")
|
|
for p in re.split(r"(?<!\\),", value)]
|
|
if len(parts) < 3:
|
|
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
|
|
thisbuild = {}
|
|
thisbuild['origlines'] = lines
|
|
thisbuild['version'] = parts[0]
|
|
thisbuild['vercode'] = parts[1]
|
|
thisbuild['commit'] = parts[2]
|
|
for p in parts[3:]:
|
|
pk, pv = p.split('=', 1)
|
|
thisbuild[pk] = pv
|
|
return thisbuild
|
|
|
|
def add_comments(key):
|
|
for comment in curcomments:
|
|
thisinfo['comments'].append((key, comment))
|
|
del curcomments[:]
|
|
|
|
thisinfo = {}
|
|
if metafile:
|
|
if not isinstance(metafile, file):
|
|
metafile = open(metafile, "r")
|
|
thisinfo['id'] = metafile.name[9:-4]
|
|
if kw.get("verbose", False):
|
|
print "Reading metadata for " + thisinfo['id']
|
|
else:
|
|
thisinfo['id'] = None
|
|
|
|
# Defaults for fields that come from metadata...
|
|
thisinfo['Name'] = None
|
|
thisinfo['Category'] = 'None'
|
|
thisinfo['Description'] = []
|
|
thisinfo['Summary'] = ''
|
|
thisinfo['License'] = 'Unknown'
|
|
thisinfo['Web Site'] = ''
|
|
thisinfo['Source Code'] = ''
|
|
thisinfo['Issue Tracker'] = ''
|
|
thisinfo['Donate'] = None
|
|
thisinfo['FlattrID'] = None
|
|
thisinfo['Bitcoin'] = None
|
|
thisinfo['Disabled'] = None
|
|
thisinfo['AntiFeatures'] = None
|
|
thisinfo['Update Check Mode'] = 'Market'
|
|
thisinfo['Auto Update Mode'] = 'None'
|
|
thisinfo['Current Version'] = ''
|
|
thisinfo['Current Version Code'] = '0'
|
|
thisinfo['Repo Type'] = ''
|
|
thisinfo['Repo'] = ''
|
|
thisinfo['Requires Root'] = False
|
|
|
|
# General defaults...
|
|
thisinfo['builds'] = []
|
|
thisinfo['comments'] = []
|
|
|
|
if metafile is None:
|
|
return thisinfo
|
|
|
|
mode = 0
|
|
buildlines = []
|
|
curcomments = []
|
|
|
|
for line in metafile:
|
|
line = line.rstrip('\r\n')
|
|
if mode == 0:
|
|
if len(line) == 0:
|
|
continue
|
|
if line.startswith("#"):
|
|
curcomments.append(line)
|
|
continue
|
|
index = line.find(':')
|
|
if index == -1:
|
|
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
|
|
field = line[:index]
|
|
value = line[index+1:]
|
|
|
|
# Translate obsolete fields...
|
|
if field == 'Market Version':
|
|
field = 'Current Version'
|
|
if field == 'Market Version Code':
|
|
field = 'Current Version Code'
|
|
|
|
fieldtype = metafieldtype(field)
|
|
if fieldtype != 'build':
|
|
add_comments(field)
|
|
if fieldtype == 'multiline':
|
|
mode = 1
|
|
thisinfo[field] = []
|
|
if len(value) > 0:
|
|
raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
|
|
elif fieldtype == 'string':
|
|
thisinfo[field] = value
|
|
elif fieldtype == 'flag':
|
|
if value == 'Yes':
|
|
thisinfo[field] = True
|
|
elif value == 'No':
|
|
thisinfo[field] = False
|
|
else:
|
|
raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name)
|
|
elif fieldtype == 'build':
|
|
if value.endswith("\\"):
|
|
mode = 2
|
|
buildlines = [value[:-1]]
|
|
else:
|
|
thisinfo['builds'].append(parse_buildline([value]))
|
|
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
|
elif fieldtype == 'obsolete':
|
|
pass # Just throw it away!
|
|
else:
|
|
raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
|
|
elif mode == 1: # Multiline field
|
|
if line == '.':
|
|
mode = 0
|
|
else:
|
|
thisinfo[field].append(line)
|
|
elif mode == 2: # Line continuation mode in Build Version
|
|
if line.endswith("\\"):
|
|
buildlines.append(line[:-1])
|
|
else:
|
|
buildlines.append(line)
|
|
thisinfo['builds'].append(
|
|
parse_buildline(buildlines))
|
|
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
|
mode = 0
|
|
add_comments(None)
|
|
|
|
# Mode at end of file should always be 0...
|
|
if mode == 1:
|
|
raise MetaDataException(field + " not terminated in " + metafile.name)
|
|
elif mode == 2:
|
|
raise MetaDataException("Unterminated continuation in " + metafile.name)
|
|
|
|
if len(thisinfo['Description']) == 0:
|
|
thisinfo['Description'].append('No description available')
|
|
|
|
# Ensure all AntiFeatures are recognised...
|
|
if thisinfo['AntiFeatures']:
|
|
parts = thisinfo['AntiFeatures'].split(",")
|
|
for part in parts:
|
|
if (part != "Ads" and
|
|
part != "Tracking" and
|
|
part != "NonFreeNet" and
|
|
part != "NonFreeDep" and
|
|
part != "NonFreeAdd"):
|
|
raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
|
|
+ metafile.name)
|
|
|
|
return thisinfo
|
|
|
|
# Write a metadata file.
|
|
#
|
|
# 'dest' - The path to the output file
|
|
# 'app' - The app data
|
|
def write_metadata(dest, app):
|
|
|
|
def writecomments(key):
|
|
for pf, comment in app['comments']:
|
|
if pf == key:
|
|
mf.write(comment + '\n')
|
|
|
|
def writefield(field, value=None):
|
|
writecomments(field)
|
|
if value is None:
|
|
value = app[field]
|
|
mf.write(field + ':' + value + '\n')
|
|
|
|
mf = open(dest, 'w')
|
|
if app['Disabled']:
|
|
writefield('Disabled')
|
|
if app['AntiFeatures']:
|
|
writefield('AntiFeatures')
|
|
writefield('Category')
|
|
writefield('License')
|
|
writefield('Web Site')
|
|
writefield('Source Code')
|
|
writefield('Issue Tracker')
|
|
if app['Donate']:
|
|
writefield('Donate')
|
|
if app['FlattrID']:
|
|
writefield('FlattrID')
|
|
if app['Bitcoin']:
|
|
writefield('Bitcoin')
|
|
mf.write('\n')
|
|
if app['Name']:
|
|
writefield('Name')
|
|
writefield('Summary')
|
|
writefield('Description', '')
|
|
for line in app['Description']:
|
|
mf.write(line + '\n')
|
|
mf.write('.\n')
|
|
mf.write('\n')
|
|
if app['Requires Root']:
|
|
writefield('Requires Root', 'Yes')
|
|
mf.write('\n')
|
|
if len(app['Repo Type']) > 0:
|
|
writefield('Repo Type')
|
|
writefield('Repo')
|
|
mf.write('\n')
|
|
for build in app['builds']:
|
|
writecomments('build:' + build['version'])
|
|
mf.write('Build Version:')
|
|
if build.has_key('origlines'):
|
|
# Keeping the original formatting if we loaded it from a file...
|
|
mf.write('\\\n'.join(build['origlines']) + '\n')
|
|
else:
|
|
mf.write(build['version'] + ',' + build['vercode'] + ',' +
|
|
build['commit'])
|
|
for key,value in build.iteritems():
|
|
if key not in ['version', 'vercode', 'commit']:
|
|
mf.write(',' + key + '=' + value)
|
|
mf.write('\n')
|
|
if len(app['builds']) > 0:
|
|
mf.write('\n')
|
|
writefield('Auto Update Mode')
|
|
writefield('Update Check Mode')
|
|
if len(app['Current Version']) > 0:
|
|
writefield('Current Version')
|
|
writefield('Current Version Code')
|
|
mf.write('\n')
|
|
writecomments(None)
|
|
mf.close()
|
|
|
|
|
|
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
|
|
# returned by the parse_metadata function.
|
|
def read_metadata(verbose=False, xref=True):
|
|
apps = []
|
|
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
|
|
if verbose:
|
|
print "Reading " + metafile
|
|
apps.append(parse_metadata(metafile, verbose=verbose))
|
|
|
|
if xref:
|
|
# Parse all descriptions at load time, just to ensure cross-referencing
|
|
# errors are caught early rather than when they hit the build server.
|
|
def linkres(link):
|
|
for app in apps:
|
|
if app['id'] == link:
|
|
return ("fdroid.app:" + link, "Dummy name - don't know yet")
|
|
raise MetaDataException("Cannot resolve app id " + link)
|
|
for app in apps:
|
|
try:
|
|
description_html(app['Description'], linkres)
|
|
except Exception, e:
|
|
raise MetaDataException("Problem with description of " + app['id'] +
|
|
" - " + str(e))
|
|
|
|
return apps
|
|
|
|
# Formatter for descriptions. Create an instance, and call parseline() with
|
|
# each line of the description source from the metadata. At the end, call
|
|
# end() and then text_plain, text_wiki and text_html will contain the result.
|
|
class DescriptionFormatter:
|
|
stNONE = 0
|
|
stPARA = 1
|
|
stUL = 2
|
|
stOL = 3
|
|
bold = False
|
|
ital = False
|
|
state = stNONE
|
|
text_plain = ''
|
|
text_wiki = ''
|
|
text_html = ''
|
|
linkResolver = None
|
|
def __init__(self, linkres):
|
|
self.linkResolver = linkres
|
|
def endcur(self, notstates=None):
|
|
if notstates and self.state in notstates:
|
|
return
|
|
if self.state == self.stPARA:
|
|
self.endpara()
|
|
elif self.state == self.stUL:
|
|
self.endul()
|
|
elif self.state == self.stOL:
|
|
self.endol()
|
|
def endpara(self):
|
|
self.text_plain += '\n'
|
|
self.text_html += '</p>'
|
|
self.state = self.stNONE
|
|
def endul(self):
|
|
self.text_html += '</ul>'
|
|
self.state = self.stNONE
|
|
def endol(self):
|
|
self.text_html += '</ol>'
|
|
self.state = self.stNONE
|
|
|
|
def formatted(self, txt, html):
|
|
formatted = ''
|
|
if html:
|
|
txt = cgi.escape(txt)
|
|
while True:
|
|
index = txt.find("''")
|
|
if index == -1:
|
|
return formatted + txt
|
|
formatted += txt[:index]
|
|
txt = txt[index:]
|
|
if txt.startswith("'''"):
|
|
if html:
|
|
if self.bold:
|
|
formatted += '</b>'
|
|
else:
|
|
formatted += '<b>'
|
|
self.bold = not self.bold
|
|
txt = txt[3:]
|
|
else:
|
|
if html:
|
|
if self.ital:
|
|
formatted += '</i>'
|
|
else:
|
|
formatted += '<i>'
|
|
self.ital = not self.ital
|
|
txt = txt[2:]
|
|
|
|
|
|
def linkify(self, txt):
|
|
linkified_plain = ''
|
|
linkified_html = ''
|
|
while True:
|
|
index = txt.find("[")
|
|
if index == -1:
|
|
return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
|
|
linkified_plain += self.formatted(txt[:index], False)
|
|
linkified_html += self.formatted(txt[:index], True)
|
|
txt = txt[index:]
|
|
if txt.startswith("[["):
|
|
index = txt.find("]]")
|
|
if index == -1:
|
|
raise MetaDataException("Unterminated ]]")
|
|
url = txt[2:index]
|
|
if self.linkResolver:
|
|
url, urltext = self.linkResolver(url)
|
|
else:
|
|
urltext = url
|
|
linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
|
|
linkified_plain += urltext
|
|
txt = txt[index+2:]
|
|
else:
|
|
index = txt.find("]")
|
|
if index == -1:
|
|
raise MetaDataException("Unterminated ]")
|
|
url = txt[1:index]
|
|
index2 = url.find(' ')
|
|
if index2 == -1:
|
|
urltxt = url
|
|
else:
|
|
urltxt = url[index2 + 1:]
|
|
url = url[:index2]
|
|
linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
|
|
linkified_plain += urltxt
|
|
if urltxt != url:
|
|
linkified_plain += ' (' + url + ')'
|
|
txt = txt[index+1:]
|
|
|
|
def addtext(self, txt):
|
|
p, h = self.linkify(txt)
|
|
self.text_plain += p
|
|
self.text_html += h
|
|
|
|
def parseline(self, line):
|
|
self.text_wiki += line + '\n'
|
|
if len(line) == 0:
|
|
self.endcur()
|
|
elif line.startswith('*'):
|
|
self.endcur([self.stUL])
|
|
if self.state != self.stUL:
|
|
self.text_html += '<ul>'
|
|
self.state = self.stUL
|
|
self.text_html += '<li>'
|
|
self.text_plain += '*'
|
|
self.addtext(line[1:])
|
|
self.text_html += '</li>'
|
|
elif line.startswith('#'):
|
|
self.endcur([self.stOL])
|
|
if self.state != self.stOL:
|
|
self.text_html += '<ol>'
|
|
self.state = self.stOL
|
|
self.text_html += '<li>'
|
|
self.text_plain += '*' #TODO: lazy - put the numbers in!
|
|
self.addtext(line[1:])
|
|
self.text_html += '</li>'
|
|
else:
|
|
self.endcur([self.stPARA])
|
|
if self.state == self.stNONE:
|
|
self.text_html += '<p>'
|
|
self.state = self.stPARA
|
|
elif self.state == self.stPARA:
|
|
self.text_html += ' '
|
|
self.text_plain += ' '
|
|
self.addtext(line)
|
|
|
|
def end(self):
|
|
self.endcur()
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in plain text format.
|
|
def description_plain(lines, linkres):
|
|
ps = DescriptionFormatter(linkres)
|
|
for line in lines:
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_plain
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in wiki format.
|
|
def description_wiki(lines):
|
|
ps = DescriptionFormatter(None)
|
|
for line in lines:
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_wiki
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in HTML format.
|
|
def description_html(lines,linkres):
|
|
ps = DescriptionFormatter(linkres)
|
|
for line in lines:
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_html
|
|
|
|
|
|
|
|
# Extract some information from the AndroidManifest.xml at the given path.
|
|
# Returns (version, vercode, package), any or all of which might be None.
|
|
# All values returned are strings.
|
|
def parse_androidmanifest(manifest):
|
|
|
|
vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search
|
|
vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search
|
|
psearch = re.compile(r'.*package="([^"]+)".*').search
|
|
version = None
|
|
vercode = None
|
|
package = None
|
|
for line in file(manifest):
|
|
if not package:
|
|
matches = psearch(line)
|
|
if matches:
|
|
package = matches.group(1)
|
|
if not version:
|
|
matches = vnsearch(line)
|
|
if matches:
|
|
version = matches.group(1)
|
|
if not vercode:
|
|
matches = vcsearch(line)
|
|
if matches:
|
|
vercode = matches.group(1)
|
|
return (version, vercode, package)
|
|
|
|
|
|
class BuildException(Exception):
|
|
def __init__(self, value, stdout = None, stderr = None):
|
|
self.value = value
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
|
|
def __str__(self):
|
|
ret = repr(self.value)
|
|
if self.stdout:
|
|
ret = ret + "\n==== stdout begin ====\n" + str(self.stdout) + "\n==== stdout end ===="
|
|
if self.stderr:
|
|
ret = ret + "\n==== stderr begin ====\n" + str(self.stderr) + "\n==== stderr end ===="
|
|
return ret
|
|
|
|
class VCSException(Exception):
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def __str__(self):
|
|
return repr(self.value)
|
|
|
|
class MetaDataException(Exception):
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def __str__(self):
|
|
return repr(self.value)
|
|
|
|
|
|
# Get the specified source library.
|
|
# Returns the path to it. Normally this is the path to be used when referencing
|
|
# it, which may be a subdirectory of the actual project. If you want the base
|
|
# directory of the project, pass 'basepath=True'.
|
|
# TODO: These are currently just hard-coded in this method. It will be a
|
|
# metadata-driven system eventually, but not yet.
|
|
def getsrclib(spec, extlib_dir, sdk_path, basepath=False):
|
|
|
|
name, ref = spec.split('@')
|
|
|
|
if name == 'GreenDroid':
|
|
sdir = os.path.join(extlib_dir, 'GreenDroid')
|
|
vcs = getvcs('git',
|
|
'https://github.com/cyrilmottier/GreenDroid.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
return os.path.join(sdir, 'GreenDroid')
|
|
|
|
if name == 'Dropbear':
|
|
sdir = os.path.join(extlib_dir, 'Dropbear')
|
|
vcs = getvcs('git',
|
|
'https://github.com/CyanogenMod/android_external_dropbear.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
return sdir
|
|
|
|
if name == 'ActionBarSherlock':
|
|
sdir = os.path.join(extlib_dir, 'ActionBarSherlock')
|
|
vcs = getvcs('git',
|
|
'https://github.com/JakeWharton/ActionBarSherlock.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'library')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating ActionBarSherlock project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'Amazing-ListView':
|
|
sdir = os.path.join(extlib_dir, 'Amazing-ListView')
|
|
vcs = getvcs('git-svn',
|
|
'http://android-amazing-listview.googlecode.com/svn/trunk', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'AmazingListView')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating Amazing-ListView project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'ViewPagerIndicator':
|
|
sdir = os.path.join(extlib_dir, 'ViewPagerIndicator')
|
|
vcs = getvcs('git',
|
|
'https://github.com/JakeWharton/Android-ViewPagerIndicator.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'library')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating Android-ViewPagerIndicator project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'UITableView':
|
|
sdir = os.path.join(extlib_dir, 'UITableView')
|
|
vcs = getvcs('git',
|
|
'https://github.com/thiagolocatelli/android-uitableview.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'android-uitableview')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating UITableView project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'ViewPagerTabs':
|
|
sdir = os.path.join(extlib_dir, 'ViewPagerTabs')
|
|
vcs = getvcs('git',
|
|
'https://github.com/astuetz/android-viewpagertabs.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
pp = open(os.path.join(sdir, 'project.properties'), 'w')
|
|
pp.write('android.library=true\n')
|
|
pp.write('target=android-15\n')
|
|
pp.close()
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
sdir]) != 0:
|
|
raise BuildException('Error updating ViewPagerTabs project')
|
|
return sdir
|
|
|
|
if name == 'ActionBar':
|
|
sdir = os.path.join(extlib_dir, 'ActionBar')
|
|
vcs = getvcs('git',
|
|
'https://github.com/johannilsson/android-actionbar.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'actionbar')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating ActionBar project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'ActionBarNW':
|
|
sdir = os.path.join(extlib_dir, 'ActionBarNW')
|
|
vcs = getvcs('git',
|
|
'https://github.com/NightWhistler/android-actionbar.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'actionbar')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating ActionBarNW project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'FacebookSDK':
|
|
sdir = os.path.join(extlib_dir, 'FacebookSDK')
|
|
vcs = getvcs('git',
|
|
'git://github.com/facebook/facebook-android-sdk.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'facebook')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating FacebookSDK project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'OI':
|
|
sdir = os.path.join(extlib_dir, 'OI')
|
|
vcs = getvcs('git-svn',
|
|
'http://openintents.googlecode.com/svn/trunk/', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
return sdir
|
|
|
|
if name == 'JOpenDocument':
|
|
sdir = os.path.join(extlib_dir, 'JOpenDocument')
|
|
vcs = getvcs('git',
|
|
'https://github.com/andiwand/JOpenDocument.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
shutil.rmtree(os.path.join(sdir, 'bin'))
|
|
return sdir
|
|
|
|
if name == 'BitcoinJWallet':
|
|
sdir = os.path.join(extlib_dir, 'BitcoinJWallet')
|
|
vcs = getvcs('git',
|
|
'https://code.google.com/r/andreasschildbach-bitcoinj/', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
return sdir
|
|
|
|
if name == 'Color-Picker':
|
|
sdir = os.path.join(extlib_dir, 'Color-Picker')
|
|
vcs = getvcs('git',
|
|
'https://github.com/brk3/android-color-picker.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
sdir]) != 0:
|
|
raise BuildException('Error updating Color-Picker project')
|
|
return sdir
|
|
|
|
if name == 'Processing-Multitouch':
|
|
sdir = os.path.join(extlib_dir, 'Processing-Multitouch')
|
|
vcs = getvcs('git',
|
|
'https://github.com/rjmarsan/AndroidProcessingMultitouch.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
sdir]) != 0:
|
|
raise BuildException('Error updating Multitouch project')
|
|
return sdir
|
|
|
|
if name == 'NewQuickAction3D':
|
|
sdir = os.path.join(extlib_dir, 'NewQuickAction3D')
|
|
vcs = getvcs('git',
|
|
'https://github.com/alt236/NewQuickAction3D.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
sdir]) != 0:
|
|
raise BuildException('Error updating NewQuickAction3D project')
|
|
return sdir
|
|
|
|
if name == 'AnySoftKeyboard-API':
|
|
sdir = os.path.join(extlib_dir, 'AnySoftKeyboard-API')
|
|
vcs = getvcs('git',
|
|
'https://github.com/AnySoftKeyboard/AnySoftKeyboard-API.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
sdir]) != 0:
|
|
raise BuildException('Error updating AnySoftKeyboard-API project')
|
|
return sdir
|
|
|
|
if name == 'File-Picker':
|
|
sdir = os.path.join(extlib_dir, 'File-Picker')
|
|
vcs = getvcs('hg',
|
|
'https://code.google.com/p/android-file-picker/', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
sdir]) != 0:
|
|
raise BuildException('Error updating File-Picker project')
|
|
return sdir
|
|
|
|
if name == 'EmulatorView':
|
|
sdir = os.path.join(extlib_dir, 'EmulatorView')
|
|
vcs = getvcs('git',
|
|
'https://github.com/jackpal/Android-Terminal-Emulator.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'libraries', 'emulatorview')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating EmulatorView project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
#Leave the native code as a blob; submodules required for that
|
|
if name == 'Libpd':
|
|
sdir = os.path.join(extlib_dir, 'Libpd')
|
|
vcs = getvcs('git',
|
|
'https://github.com/libpd/pd-for-android.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'PdCore')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('LibPd project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'Tree-View-List':
|
|
sdir = os.path.join(extlib_dir, 'Tree-View-List')
|
|
vcs = getvcs('hg',
|
|
'https://code.google.com/p/tree-view-list-android/', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
sdir]) != 0:
|
|
raise BuildException('Error updating Tree-List-View project')
|
|
return sdir
|
|
|
|
if name == 'PrayTimes':
|
|
sdir = os.path.join(extlib_dir, 'PrayTimes')
|
|
vcs = getvcs('git',
|
|
'https://github.com/ebraminio/PrayTimes.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
return sdir
|
|
|
|
if name == 'HoloEverywhere':
|
|
sdir = os.path.join(extlib_dir, 'HoloEverywhere')
|
|
vcs = getvcs('git',
|
|
'https://github.com/ChristopheVersieux/HoloEverywhere.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'library')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating HoloEverywhere project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'PullToRefresh':
|
|
sdir = os.path.join(extlib_dir, 'PullToRefresh')
|
|
vcs = getvcs('git',
|
|
'https://github.com/chrisbanes/Android-PullToRefresh.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'library')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating PullToRefresh project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'TessTwo':
|
|
sdir = os.path.join(extlib_dir, 'TessTwo')
|
|
vcs = getvcs('git',
|
|
'https://github.com/rmtheis/tess-two.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
libdir = os.path.join(sdir, 'tess-two')
|
|
if subprocess.call([os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p',
|
|
libdir]) != 0:
|
|
raise BuildException('Error updating TessTwo project')
|
|
if basepath:
|
|
return sdir
|
|
return libdir
|
|
|
|
if name == 'TwidereExtension':
|
|
sdir = os.path.join(extlib_dir, 'TwidereExtension')
|
|
vcs = getvcs('git',
|
|
'https://github.com/mariotaku/twidere-extension-library.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
return sdir
|
|
|
|
if name == 'EncfsJava':
|
|
sdir = os.path.join(extlib_dir, 'EncfsJava')
|
|
vcs = getvcs('git',
|
|
'https://github.com/mrpdaemon/encfs-java.git', sdir, sdk_path)
|
|
vcs.gotorevision(ref)
|
|
return sdir
|
|
|
|
raise BuildException('Unknown srclib ' + name)
|
|
|
|
|
|
# Prepare the source code for a particular build
|
|
# 'vcs' - the appropriate vcs object for the application
|
|
# 'app' - the application details from the metadata
|
|
# 'build' - the build details from the metadata
|
|
# 'build_dir' - the path to the build directory, usually
|
|
# 'build/app.id'
|
|
# 'extlib_dir' - the path to the external libraries directory, usually
|
|
# 'build/extlib'
|
|
# 'sdk_path' - the path to the Android SDK
|
|
# 'ndk_path' - the path to the Android NDK
|
|
# 'javacc_path' - the path to javacc
|
|
# 'mvn3' - the path to the maven 3 executable
|
|
# 'verbose' - optional: verbose or not (default=False)
|
|
# Returns the root directory, which may be the same as 'build_dir' or may
|
|
# be a subdirectory of it.
|
|
def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, verbose=False):
|
|
|
|
# Optionally, the actual app source can be in a subdirectory...
|
|
if build.has_key('subdir'):
|
|
root_dir = os.path.join(build_dir, build['subdir'])
|
|
else:
|
|
root_dir = build_dir
|
|
|
|
# Get a working copy of the right revision...
|
|
print "Getting source for revision " + build['commit']
|
|
vcs.gotorevision(build['commit'])
|
|
|
|
# Check that a subdir (if we're using one) exists. This has to happen
|
|
# after the checkout, since it might not exist elsewhere...
|
|
if not os.path.exists(root_dir):
|
|
raise BuildException('Missing subdir ' + root_dir)
|
|
|
|
# Initialise submodules if requred...
|
|
if build.get('submodules', 'no') == 'yes':
|
|
if verbose: print "Initialising submodules..."
|
|
vcs.initsubmodules()
|
|
|
|
# Run an init command if one is required...
|
|
if build.has_key('init'):
|
|
init = build['init']
|
|
init = init.replace('$$SDK$$', sdk_path)
|
|
init = init.replace('$$NDK$$', ndk_path)
|
|
init = init.replace('$$MVN$$', mvn3)
|
|
if verbose: print "Doing init: exec '%s' in '%s'"%(init,root_dir)
|
|
if subprocess.call(init, cwd=root_dir, shell=True) != 0:
|
|
raise BuildException("Error running init command")
|
|
|
|
# Generate (or update) the ant build file, build.xml...
|
|
updatemode = build.get('update', '.')
|
|
if (updatemode != 'no' and
|
|
not build.has_key('maven')):
|
|
parms = [os.path.join(sdk_path, 'tools', 'android'),
|
|
'update', 'project', '-p', '.']
|
|
parms.append('--subprojects')
|
|
if build.has_key('target'):
|
|
parms.append('-t')
|
|
parms.append(build['target'])
|
|
update_dirs = updatemode.split(';')
|
|
# Force build.xml update if necessary...
|
|
if updatemode == 'force' or build.has_key('target'):
|
|
if updatemode == 'force':
|
|
update_dirs = ['.']
|
|
buildxml = os.path.join(root_dir, 'build.xml')
|
|
if os.path.exists(buildxml):
|
|
print 'Force-removing old build.xml'
|
|
os.remove(buildxml)
|
|
for d in update_dirs:
|
|
cwd = root_dir + '/' + d
|
|
if verbose:
|
|
print "Update of '%s': exec '%s' in '%s'"%\
|
|
(d," ".join(parms),cwd)
|
|
p = subprocess.Popen(parms, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
(out, err) = p.communicate()
|
|
if p.returncode != 0:
|
|
raise BuildException("Failed to update project with stdout '%s' and stderr '%s'"%(out,err))
|
|
# check to see whether an error was returned without a proper exit code (this is the case for the 'no target set or target invalid' error)
|
|
if err != "" and err.startswith("Error: "):
|
|
raise BuildException("Failed to update project with stdout '%s' and stderr '%s'"%(out,err))
|
|
|
|
# If the app has ant set up to sign the release, we need to switch
|
|
# that off, because we want the unsigned apk...
|
|
for propfile in ('build.properties', 'default.properties', 'ant.properties'):
|
|
if os.path.exists(os.path.join(root_dir, propfile)):
|
|
if subprocess.call(['sed','-i','s/^key.store/#/',
|
|
propfile], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend %s" % propfile)
|
|
|
|
# Update the local.properties file...
|
|
locprops = os.path.join(root_dir, 'local.properties')
|
|
if os.path.exists(locprops):
|
|
f = open(locprops, 'r')
|
|
props = f.read()
|
|
f.close()
|
|
# Fix old-fashioned 'sdk-location' by copying
|
|
# from sdk.dir, if necessary...
|
|
if build.get('oldsdkloc', 'no') == "yes":
|
|
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
|
|
re.S|re.M).group(1)
|
|
props += "\nsdk-location=" + sdkloc + "\n"
|
|
# Add ndk location...
|
|
props+= "\nndk.dir=" + ndk_path + "\n"
|
|
# Add java.encoding if necessary...
|
|
if build.has_key('encoding'):
|
|
props += "\njava.encoding=" + build['encoding'] + "\n"
|
|
f = open(locprops, 'w')
|
|
f.write(props)
|
|
f.close()
|
|
|
|
# Insert version code and number into the manifest if necessary...
|
|
if build.has_key('forceversion'):
|
|
if subprocess.call(['sed','-r','-i',
|
|
's/android:versionName="[^"]+"/android:versionName="' + build['version'] + '"/g',
|
|
'AndroidManifest.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend manifest")
|
|
if build.has_key('forcevercode'):
|
|
if subprocess.call(['sed','-r','-i',
|
|
's/android:versionCode="[^"]+"/android:versionCode="' + build['vercode'] + '"/g',
|
|
'AndroidManifest.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend manifest")
|
|
|
|
# Delete unwanted file...
|
|
if build.has_key('rm'):
|
|
dest = os.path.join(build_dir, build['rm'])
|
|
if os.path.exists(dest):
|
|
os.remove(dest)
|
|
|
|
# Fix apostrophes translation files if necessary...
|
|
if build.get('fixapos', 'no') == 'yes':
|
|
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
|
|
for filename in files:
|
|
if filename.endswith('.xml'):
|
|
if subprocess.call(['sed','-i','s@' +
|
|
r"\([^\\]\)'@\1\\'" +
|
|
'@g',
|
|
os.path.join(root, filename)]) != 0:
|
|
raise BuildException("Failed to amend " + filename)
|
|
|
|
# Fix translation files if necessary...
|
|
if build.get('fixtrans', 'no') == 'yes':
|
|
for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
|
|
for filename in files:
|
|
if filename.endswith('.xml'):
|
|
f = open(os.path.join(root, filename))
|
|
changed = False
|
|
outlines = []
|
|
for line in f:
|
|
num = 1
|
|
index = 0
|
|
oldline = line
|
|
while True:
|
|
index = line.find("%", index)
|
|
if index == -1:
|
|
break
|
|
next = line[index+1:index+2]
|
|
if next == "s" or next == "d":
|
|
line = (line[:index+1] +
|
|
str(num) + "$" +
|
|
line[index+1:])
|
|
num += 1
|
|
index += 3
|
|
else:
|
|
index += 1
|
|
# We only want to insert the positional arguments
|
|
# when there is more than one argument...
|
|
if oldline != line:
|
|
if num > 2:
|
|
changed = True
|
|
else:
|
|
line = oldline
|
|
outlines.append(line)
|
|
f.close()
|
|
if changed:
|
|
f = open(os.path.join(root, filename), 'w')
|
|
f.writelines(outlines)
|
|
f.close()
|
|
|
|
# Add required external libraries...
|
|
if build.has_key('extlibs'):
|
|
libsdir = os.path.join(root_dir, 'libs')
|
|
if not os.path.exists(libsdir):
|
|
os.mkdir(libsdir)
|
|
for lib in build['extlibs'].split(';'):
|
|
libf = os.path.basename(lib)
|
|
shutil.copyfile(os.path.join(extlib_dir, lib),
|
|
os.path.join(libsdir, libf))
|
|
|
|
# Get required source libraries...
|
|
srclibpaths = []
|
|
if build.has_key('srclibs'):
|
|
for lib in build['srclibs'].split(';'):
|
|
name, _ = lib.split('@')
|
|
srclibpaths.append((name, getsrclib(lib, extlib_dir, sdk_path)))
|
|
basesrclib = vcs.getsrclib()
|
|
# If one was used for the main source, add that too.
|
|
if basesrclib:
|
|
srclibpaths.append(basesrclib)
|
|
|
|
# There should never be gen or bin directories in the source, so just get
|
|
# rid of them...
|
|
for baddir in ['gen', 'bin']:
|
|
badpath = os.path.join(root_dir, baddir)
|
|
if os.path.exists(badpath):
|
|
shutil.rmtree(badpath)
|
|
|
|
# Apply patches if any
|
|
if 'patch' in build:
|
|
for patch in build['patch'].split(';'):
|
|
print "Applying " + patch
|
|
patch_path = os.path.join('metadata', app['id'], patch)
|
|
if subprocess.call(['patch', '-p1',
|
|
'-i', os.path.abspath(patch_path)], cwd=build_dir) != 0:
|
|
raise BuildException("Failed to apply patch %s" % patch_path)
|
|
|
|
# Run a pre-build command if one is required...
|
|
if build.has_key('prebuild'):
|
|
prebuild = build['prebuild']
|
|
# Substitute source library paths into prebuild commands...
|
|
for name, libpath in srclibpaths:
|
|
libpath = os.path.relpath(libpath, root_dir)
|
|
prebuild = prebuild.replace('$$' + name + '$$', libpath)
|
|
prebuild = prebuild.replace('$$SDK$$', sdk_path)
|
|
prebuild = prebuild.replace('$$NDK$$', ndk_path)
|
|
prebuild = prebuild.replace('$$MVN3$$', mvn3)
|
|
p = subprocess.Popen(prebuild, cwd=root_dir, shell=True,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
out, err = p.communicate()
|
|
if p.returncode != 0:
|
|
raise BuildException("Error running pre-build command", out, err)
|
|
|
|
if build.get('anal-tics', 'no') == 'yes':
|
|
fp = os.path.join(root_dir, 'src', 'com', 'google', 'android', 'apps', 'analytics')
|
|
os.makedirs(fp)
|
|
with open(os.path.join(fp, 'GoogleAnalyticsTracker.java'), 'w') as f:
|
|
f.write("""
|
|
package com.google.android.apps.analytics;
|
|
public class GoogleAnalyticsTracker {
|
|
private static GoogleAnalyticsTracker instance;
|
|
private GoogleAnalyticsTracker() {
|
|
}
|
|
public static GoogleAnalyticsTracker getInstance() {
|
|
if(instance == null)
|
|
instance = new GoogleAnalyticsTracker();
|
|
return instance;
|
|
}
|
|
public void start(String i,int think ,Object not) {
|
|
}
|
|
public void dispatch() {
|
|
}
|
|
public void stop() {
|
|
}
|
|
public void setProductVersion(String uh, String hu) {
|
|
}
|
|
public void trackEvent(String that,String just,String aint,int happening) {
|
|
}
|
|
public void trackPageView(String nope) {
|
|
}
|
|
public void setCustomVar(int mind,String your,String own,int business) {
|
|
}
|
|
}
|
|
""")
|
|
|
|
# Special case init functions for funambol...
|
|
if build.get('initfun', 'no') == "yes":
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'<taskdef resource="net/sf/antcontrib/antcontrib.properties" />' +
|
|
'@' +
|
|
'<taskdef resource="net/sf/antcontrib/antcontrib.properties">' +
|
|
'<classpath>' +
|
|
'<pathelement location="/usr/share/java/ant-contrib.jar"/>' +
|
|
'</classpath>' +
|
|
'</taskdef>' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.xml")
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'\${user.home}/funambol/build/android/build.properties' +
|
|
'@' +
|
|
'build.properties' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.xml")
|
|
|
|
buildxml = os.path.join(root_dir, 'build.xml')
|
|
f = open(buildxml, 'r')
|
|
xml = f.read()
|
|
f.close()
|
|
xmlout = ""
|
|
mode = 0
|
|
for line in xml.splitlines():
|
|
if mode == 0:
|
|
if line.find("jarsigner") != -1:
|
|
mode = 1
|
|
else:
|
|
xmlout += line + "\n"
|
|
else:
|
|
if line.find("/exec") != -1:
|
|
mode += 1
|
|
if mode == 3:
|
|
mode =0
|
|
f = open(buildxml, 'w')
|
|
f.write(xmlout)
|
|
f.close()
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'platforms/android-2.0' +
|
|
'@' +
|
|
'platforms/android-8' +
|
|
'@g',
|
|
'build.xml'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.xml")
|
|
|
|
shutil.copyfile(
|
|
os.path.join(root_dir, "build.properties.example"),
|
|
os.path.join(root_dir, "build.properties"))
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'javacchome=.*'+
|
|
'@' +
|
|
'javacchome=' + javacc_path +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.properties")
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'sdk-folder=.*'+
|
|
'@' +
|
|
'sdk-folder=' + sdk_path +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.properties")
|
|
|
|
if subprocess.call(['sed','-i','s@' +
|
|
'android.sdk.version.*'+
|
|
'@' +
|
|
'android.sdk.version=2.0' +
|
|
'@g',
|
|
'build.properties'], cwd=root_dir) !=0:
|
|
raise BuildException("Failed to amend build.properties")
|
|
|
|
return root_dir
|
|
|
|
|
|
# Scan the source code in the given directory (and all subdirectories)
|
|
# and return a list of potential problems.
|
|
def scan_source(build_dir, root_dir, thisbuild):
|
|
|
|
problems = []
|
|
|
|
# Common known non-free blobs:
|
|
usual_suspects = ['flurryagent',
|
|
'paypal_mpl',
|
|
'libgoogleanalytics',
|
|
'admob-sdk-android',
|
|
'googleadview',
|
|
'googleadmobadssdk',
|
|
'heyzap',
|
|
'jpct-ae']
|
|
|
|
# Iterate through all files in the source code...
|
|
for r,d,f in os.walk(build_dir):
|
|
for curfile in f:
|
|
|
|
if r.find('/.hg/') == -1:
|
|
|
|
# Path (relative) to the file...
|
|
fp = os.path.join(r, curfile)
|
|
|
|
for suspect in usual_suspects:
|
|
if curfile.lower().find(suspect) != -1:
|
|
msg = 'Found probable non-free blob ' + fp
|
|
problems.append(msg)
|
|
|
|
if curfile.endswith('.apk'):
|
|
msg = 'Found apk file, which should not be in the source - ' + fp
|
|
problems.append(msg)
|
|
|
|
if curfile.endswith('.java'):
|
|
for line in file(fp):
|
|
|
|
if line.find('DexClassLoader') != -1:
|
|
msg = 'Found DexClassLoader in ' + fp
|
|
problems.append(msg)
|
|
|
|
# Presence of a jni directory without buildjni=yes might
|
|
# indicate a problem... (if it's not a problem, explicitly use
|
|
# buildjni=no to bypass this check)
|
|
if (os.path.exists(os.path.join(root_dir, 'jni')) and
|
|
thisbuild.get('buildjni') is None):
|
|
msg = 'Found jni directory, but buildjni is not enabled'
|
|
problems.append(msg)
|
|
|
|
return problems
|
|
|
|
|
|
class KnownApks:
|
|
|
|
def __init__(self):
|
|
self.path = os.path.join('stats', 'known_apks.txt')
|
|
self.apks = {}
|
|
if os.path.exists(self.path):
|
|
for line in file( self.path):
|
|
t = line.rstrip().split(' ')
|
|
if len(t) == 2:
|
|
self.apks[t[0]] = (t[1], None)
|
|
else:
|
|
self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
|
|
self.changed = False
|
|
|
|
def writeifchanged(self):
|
|
if self.changed:
|
|
if not os.path.exists('stats'):
|
|
os.mkdir('stats')
|
|
f = open(self.path, 'w')
|
|
lst = []
|
|
for apk, app in self.apks.iteritems():
|
|
appid, added = app
|
|
line = apk + ' ' + appid
|
|
if added:
|
|
line += ' ' + time.strftime('%Y-%m-%d', added)
|
|
lst.append(line)
|
|
for line in sorted(lst):
|
|
f.write(line + '\n')
|
|
f.close()
|
|
|
|
# Record an apk (if it's new, otherwise does nothing)
|
|
# Returns the date it was added.
|
|
def recordapk(self, apk, app):
|
|
if not apk in self.apks:
|
|
self.apks[apk] = (app, time.gmtime(time.time()))
|
|
self.changed = True
|
|
_, added = self.apks[apk]
|
|
return added
|
|
|
|
# Look up information - given the 'apkname', returns (app id, date added/None).
|
|
# Or returns None for an unknown apk.
|
|
def getapp(self, apkname):
|
|
if apkname in self.apks:
|
|
return self.apks[apkname]
|
|
return None
|
|
|
|
# Get the most recent 'num' apps added to the repo, as a list of package ids
|
|
# with the most recent first.
|
|
def getlatest(self, num):
|
|
apps = {}
|
|
for apk, app in self.apks.iteritems():
|
|
appid, added = app
|
|
if added:
|
|
if appid in apps:
|
|
if apps[appid] > added:
|
|
apps[appid] = added
|
|
else:
|
|
apps[appid] = added
|
|
sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
|
|
lst = []
|
|
for app, added in sortedapps:
|
|
lst.append(app)
|
|
lst.reverse()
|
|
return lst
|
|
|